2024-11-22 23:14:05,742 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 23:14:05,754 main DEBUG Took 0.010161 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:14:05,754 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:14:05,754 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:14:05,755 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:14:05,757 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,764 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:14:05,776 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,777 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,778 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,779 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,779 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,780 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,781 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,781 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,782 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,783 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,783 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,784 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,784 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,785 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,785 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,785 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,786 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,786 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,787 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:14:05,787 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,788 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:14:05,789 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:14:05,790 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:14:05,792 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:14:05,793 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:14:05,794 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:14:05,794 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:14:05,802 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:14:05,804 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:14:05,806 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:14:05,806 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:14:05,807 main DEBUG createAppenders(={Console}) 2024-11-22 23:14:05,807 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-22 23:14:05,808 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-22 23:14:05,808 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-22 23:14:05,808 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:14:05,809 main DEBUG OutputStream closed 2024-11-22 23:14:05,809 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:14:05,809 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:14:05,809 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-22 23:14:05,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:14:05,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:14:05,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:14:05,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:14:05,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:14:05,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:14:05,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:14:05,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:14:05,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:14:05,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:14:05,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:14:05,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:14:05,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:14:05,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:14:05,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:14:05,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:14:05,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:14:05,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:14:05,899 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:14:05,899 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-22 23:14:05,899 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:14:05,900 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-22T23:14:06,096 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7 2024-11-22 23:14:06,099 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:14:06,099 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:14:06,107 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-22T23:14:06,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T23:14:06,143 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d, deleteOnExit=true 2024-11-22T23:14:06,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T23:14:06,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/test.cache.data in system properties and HBase conf 2024-11-22T23:14:06,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T23:14:06,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.log.dir in system properties and HBase conf 2024-11-22T23:14:06,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T23:14:06,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T23:14:06,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T23:14:06,235 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:14:06,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:14:06,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:14:06,328 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T23:14:06,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T23:14:06,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:14:06,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T23:14:06,330 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T23:14:06,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T23:14:06,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:14:06,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T23:14:06,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/nfs.dump.dir in system properties and HBase conf 2024-11-22T23:14:06,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/java.io.tmpdir in system properties and HBase conf 2024-11-22T23:14:06,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T23:14:06,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T23:14:06,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T23:14:07,412 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T23:14:07,482 INFO [Time-limited test {}] log.Log(170): Logging initialized @2408ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T23:14:07,549 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:14:07,604 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:14:07,629 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:14:07,629 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:14:07,631 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T23:14:07,646 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:14:07,649 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:14:07,650 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:14:07,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/java.io.tmpdir/jetty-localhost-37627-hadoop-hdfs-3_4_1-tests_jar-_-any-10816283148437431095/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:14:07,824 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:37627} 2024-11-22T23:14:07,824 INFO [Time-limited test {}] server.Server(415): Started @2752ms 2024-11-22T23:14:08,342 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T23:14:08,350 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T23:14:08,351 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T23:14:08,352 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T23:14:08,352 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T23:14:08,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.log.dir/,AVAILABLE} 2024-11-22T23:14:08,353 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T23:14:08,453 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/java.io.tmpdir/jetty-localhost-37901-hadoop-hdfs-3_4_1-tests_jar-_-any-5300655062374005564/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:14:08,454 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:37901} 2024-11-22T23:14:08,455 INFO [Time-limited test {}] server.Server(415): Started @3382ms 2024-11-22T23:14:08,504 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T23:14:09,680 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/data/data2/current/BP-579577094-172.17.0.2-1732317246869/current, will proceed with Du for space computation calculation, 2024-11-22T23:14:09,680 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/data/data1/current/BP-579577094-172.17.0.2-1732317246869/current, will proceed with Du for space computation calculation, 2024-11-22T23:14:09,714 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T23:14:09,781 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x78bda9526518b488 with lease ID 0x5a36ef1e3cbc53cc: Processing first storage report for DS-97f07e3e-a6bc-4b73-a700-ce5c4605b114 from datanode DatanodeRegistration(127.0.0.1:41203, datanodeUuid=df18387f-e6c9-4fc0-94ee-d6ae1ff9699b, infoPort=36209, infoSecurePort=0, ipcPort=34597, storageInfo=lv=-57;cid=testClusterID;nsid=644607789;c=1732317246869) 2024-11-22T23:14:09,783 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x78bda9526518b488 with lease ID 0x5a36ef1e3cbc53cc: from storage DS-97f07e3e-a6bc-4b73-a700-ce5c4605b114 node DatanodeRegistration(127.0.0.1:41203, datanodeUuid=df18387f-e6c9-4fc0-94ee-d6ae1ff9699b, infoPort=36209, infoSecurePort=0, ipcPort=34597, storageInfo=lv=-57;cid=testClusterID;nsid=644607789;c=1732317246869), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-22T23:14:09,784 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x78bda9526518b488 with lease ID 0x5a36ef1e3cbc53cc: Processing first storage report for DS-b8a2a6c8-0fb7-457c-83cf-a8ce192479c2 from datanode DatanodeRegistration(127.0.0.1:41203, datanodeUuid=df18387f-e6c9-4fc0-94ee-d6ae1ff9699b, infoPort=36209, infoSecurePort=0, ipcPort=34597, storageInfo=lv=-57;cid=testClusterID;nsid=644607789;c=1732317246869) 2024-11-22T23:14:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x78bda9526518b488 with lease ID 0x5a36ef1e3cbc53cc: from storage DS-b8a2a6c8-0fb7-457c-83cf-a8ce192479c2 node DatanodeRegistration(127.0.0.1:41203, datanodeUuid=df18387f-e6c9-4fc0-94ee-d6ae1ff9699b, infoPort=36209, infoSecurePort=0, ipcPort=34597, storageInfo=lv=-57;cid=testClusterID;nsid=644607789;c=1732317246869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-22T23:14:09,811 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7 2024-11-22T23:14:09,905 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/zookeeper_0, clientPort=62676, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:14:09,916 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62676 2024-11-22T23:14:09,930 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:09,935 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741825_1001 (size=7) 2024-11-22T23:14:10,597 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c with version=8 2024-11-22T23:14:10,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/hbase-staging 2024-11-22T23:14:10,678 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T23:14:10,873 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3d87a7361f3:0 server-side Connection retries=45 2024-11-22T23:14:10,882 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:10,882 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:10,887 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:14:10,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:10,887 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:14:11,025 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:14:11,077 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:14:11,087 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:14:11,091 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:14:11,118 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4687 (auto-detected) 2024-11-22T23:14:11,119 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T23:14:11,135 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34971 2024-11-22T23:14:11,153 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34971 connecting to ZooKeeper ensemble=127.0.0.1:62676 2024-11-22T23:14:11,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349710x0, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:14:11,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34971-0x10164d37d070000 connected 2024-11-22T23:14:11,722 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:11,726 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:11,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:14:11,739 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c, hbase.cluster.distributed=false 2024-11-22T23:14:11,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:14:11,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34971 2024-11-22T23:14:11,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34971 2024-11-22T23:14:11,765 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34971 2024-11-22T23:14:11,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34971 2024-11-22T23:14:11,766 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34971 2024-11-22T23:14:11,866 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3d87a7361f3:0 server-side Connection retries=45 2024-11-22T23:14:11,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:11,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:11,868 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:14:11,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:14:11,868 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:14:11,870 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:14:11,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:14:11,875 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46715 2024-11-22T23:14:11,877 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46715 connecting to ZooKeeper ensemble=127.0.0.1:62676 2024-11-22T23:14:11,879 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:11,883 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:11,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467150x0, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:14:11,933 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:14:11,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46715-0x10164d37d070001 connected 2024-11-22T23:14:11,938 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:14:11,948 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:14:11,951 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:14:11,955 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:14:11,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46715 2024-11-22T23:14:11,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46715 2024-11-22T23:14:11,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46715 2024-11-22T23:14:11,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46715 2024-11-22T23:14:11,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46715 2024-11-22T23:14:11,975 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3d87a7361f3:34971 2024-11-22T23:14:11,976 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:12,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:14:12,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:14:12,034 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:12,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:12,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:14:12,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:12,102 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:14:12,103 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3d87a7361f3,34971,1732317250724 from backup master directory 2024-11-22T23:14:12,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:14:12,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:12,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:14:12,145 WARN [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:14:12,145 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:12,149 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T23:14:12,151 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T23:14:12,201 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/hbase.id] with ID: 2f92b64c-3235-4726-bb52-42f565334fc6 2024-11-22T23:14:12,202 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/.tmp/hbase.id 2024-11-22T23:14:12,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741826_1002 (size=42) 2024-11-22T23:14:12,615 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/.tmp/hbase.id]:[hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/hbase.id] 2024-11-22T23:14:12,669 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T23:14:12,674 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:14:12,694 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-22T23:14:12,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:12,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741827_1003 (size=196) 2024-11-22T23:14:13,166 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:14:13,169 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:14:13,193 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:13,200 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:14:13,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741828_1004 (size=1189) 2024-11-22T23:14:13,282 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store 2024-11-22T23:14:13,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741829_1005 (size=34) 2024-11-22T23:14:13,311 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:14:13,315 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:13,316 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:14:13,316 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:14:13,317 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:14:13,319 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:14:13,320 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:14:13,320 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:14:13,322 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732317253316Disabling compacts and flushes for region at 1732317253316Disabling writes for close at 1732317253319 (+3 ms)Writing region close event to WAL at 1732317253320 (+1 ms)Closed at 1732317253320 2024-11-22T23:14:13,326 WARN [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/.initializing 2024-11-22T23:14:13,327 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/WALs/a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:13,337 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:14:13,357 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3d87a7361f3%2C34971%2C1732317250724, suffix=, logDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/WALs/a3d87a7361f3,34971,1732317250724, archiveDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/oldWALs, maxLogs=10 2024-11-22T23:14:13,383 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/WALs/a3d87a7361f3,34971,1732317250724/a3d87a7361f3%2C34971%2C1732317250724.1732317253362, exclude list is [], retry=0 2024-11-22T23:14:13,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-97f07e3e-a6bc-4b73-a700-ce5c4605b114,DISK] 2024-11-22T23:14:13,404 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T23:14:13,443 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/WALs/a3d87a7361f3,34971,1732317250724/a3d87a7361f3%2C34971%2C1732317250724.1732317253362 2024-11-22T23:14:13,444 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36209:36209)] 2024-11-22T23:14:13,445 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:14:13,445 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:13,450 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,452 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,487 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:14:13,513 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:13,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:13,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:14:13,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:13,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:13,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,527 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:14:13,528 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:13,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:13,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:14:13,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:13,535 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:13,535 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,540 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,541 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,547 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,548 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,552 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:14:13,560 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:14:13,566 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:14:13,568 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61599760, jitterRate=-0.0820920467376709}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:14:13,577 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732317253464Initializing all the Stores at 1732317253466 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317253467 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317253468 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317253468Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317253468Cleaning up temporary data from old regions at 1732317253548 (+80 ms)Region opened successfully at 1732317253576 (+28 ms) 2024-11-22T23:14:13,578 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:14:13,614 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32964c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3d87a7361f3/172.17.0.2:0 2024-11-22T23:14:13,642 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:14:13,652 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T23:14:13,653 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T23:14:13,656 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:14:13,657 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T23:14:13,664 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:14:13,664 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:14:13,692 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:14:13,700 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:14:13,728 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:14:13,731 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:14:13,734 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:14:13,749 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:14:13,751 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:14:13,755 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:14:13,759 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:14:13,762 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:14:13,770 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:14:13,787 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:14:13,791 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:14:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:14:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:14:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:13,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:13,806 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3d87a7361f3,34971,1732317250724, sessionid=0x10164d37d070000, setting cluster-up flag (Was=false) 2024-11-22T23:14:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:13,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:13,865 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:14:13,870 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:14,181 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:14:14,183 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:14,188 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:14:14,251 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:14:14,259 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:14:14,264 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(746): ClusterId : 2f92b64c-3235-4726-bb52-42f565334fc6 2024-11-22T23:14:14,265 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:14:14,266 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:14:14,270 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3d87a7361f3,34971,1732317250724 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:14:14,277 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:14:14,277 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:14:14,278 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:14:14,278 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:14:14,278 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:14:14,278 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:14:14,278 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3d87a7361f3:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:14:14,279 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,279 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:14:14,279 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,284 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732317284284 2024-11-22T23:14:14,284 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:14:14,285 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:14:14,285 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:14:14,287 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:14:14,287 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:14:14,288 DEBUG [RS:0;a3d87a7361f3:46715 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4adb595c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3d87a7361f3/172.17.0.2:0 2024-11-22T23:14:14,290 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:14:14,291 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:14,291 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:14:14,291 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:14:14,291 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:14:14,291 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:14:14,296 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,300 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:14:14,301 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:14:14,301 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:14:14,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741831_1007 (size=1321) 2024-11-22T23:14:14,304 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:14:14,304 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:14:14,306 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317254305,5,FailOnTimeoutGroup] 2024-11-22T23:14:14,308 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317254306,5,FailOnTimeoutGroup] 2024-11-22T23:14:14,308 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,308 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:14:14,309 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3d87a7361f3:46715 2024-11-22T23:14:14,309 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,310 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,312 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:14:14,312 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:14:14,312 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:14:14,314 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3d87a7361f3,34971,1732317250724 with port=46715, startcode=1732317251834 2024-11-22T23:14:14,324 DEBUG [RS:0;a3d87a7361f3:46715 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:14:14,357 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55269, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:14:14,364 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34971 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34971 {}] master.ServerManager(517): Registering regionserver=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,383 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:14:14,383 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36519 2024-11-22T23:14:14,383 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:14:14,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:14:14,392 DEBUG [RS:0;a3d87a7361f3:46715 {}] zookeeper.ZKUtil(111): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,392 WARN [RS:0;a3d87a7361f3:46715 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:14:14,392 INFO [RS:0;a3d87a7361f3:46715 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:14:14,393 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,394 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3d87a7361f3,46715,1732317251834] 2024-11-22T23:14:14,416 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:14:14,431 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:14:14,435 INFO [RS:0;a3d87a7361f3:46715 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:14:14,436 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,437 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:14:14,441 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:14:14,443 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,443 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,443 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,443 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,443 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3d87a7361f3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,444 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,445 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,445 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:14:14,445 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3d87a7361f3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:14:14,445 DEBUG [RS:0;a3d87a7361f3:46715 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,446 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,46715,1732317251834-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:14:14,465 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:14:14,466 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,46715,1732317251834-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,467 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.Replication(171): a3d87a7361f3,46715,1732317251834 started 2024-11-22T23:14:14,484 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:14,485 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1482): Serving as a3d87a7361f3,46715,1732317251834, RpcServer on a3d87a7361f3/172.17.0.2:46715, sessionid=0x10164d37d070001 2024-11-22T23:14:14,485 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:14:14,485 DEBUG [RS:0;a3d87a7361f3:46715 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,486 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3d87a7361f3,46715,1732317251834' 2024-11-22T23:14:14,486 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:14:14,487 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:14:14,488 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:14:14,488 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:14:14,488 DEBUG [RS:0;a3d87a7361f3:46715 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:14,488 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3d87a7361f3,46715,1732317251834' 2024-11-22T23:14:14,488 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:14:14,489 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:14:14,489 DEBUG [RS:0;a3d87a7361f3:46715 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:14:14,489 INFO [RS:0;a3d87a7361f3:46715 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:14:14,489 INFO [RS:0;a3d87a7361f3:46715 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:14:14,598 INFO [RS:0;a3d87a7361f3:46715 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:14:14,605 INFO [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3d87a7361f3%2C46715%2C1732317251834, suffix=, logDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834, archiveDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/oldWALs, maxLogs=32 2024-11-22T23:14:14,622 DEBUG [RS:0;a3d87a7361f3:46715 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834/a3d87a7361f3%2C46715%2C1732317251834.1732317254608, exclude list is [], retry=0 2024-11-22T23:14:14,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-97f07e3e-a6bc-4b73-a700-ce5c4605b114,DISK] 2024-11-22T23:14:14,630 INFO [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834/a3d87a7361f3%2C46715%2C1732317251834.1732317254608 2024-11-22T23:14:14,630 DEBUG [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36209:36209)] 2024-11-22T23:14:14,707 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:14:14,708 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:14:14,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741833_1009 (size=32) 2024-11-22T23:14:15,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:15,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:14:15,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:14:15,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:14:15,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:14:15,143 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:14:15,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:14:15,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:14:15,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:14:15,152 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,153 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:14:15,154 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740 2024-11-22T23:14:15,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740 2024-11-22T23:14:15,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:14:15,158 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:14:15,159 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:14:15,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:14:15,165 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:14:15,166 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60494810, jitterRate=-0.09855708479881287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:14:15,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732317255129Initializing all the Stores at 1732317255134 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255134Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255134Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317255134Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255134Cleaning up temporary data from old regions at 1732317255158 (+24 ms)Region opened successfully at 1732317255169 (+11 ms) 2024-11-22T23:14:15,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:14:15,169 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:14:15,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:14:15,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:14:15,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:14:15,171 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:14:15,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732317255169Disabling compacts and flushes for region at 1732317255169Disabling writes for close at 1732317255169Writing region close event to WAL at 1732317255170 (+1 ms)Closed at 1732317255171 (+1 ms) 2024-11-22T23:14:15,174 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:14:15,174 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:14:15,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:14:15,187 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:14:15,190 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:14:15,344 DEBUG [a3d87a7361f3:34971 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T23:14:15,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:15,363 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3d87a7361f3,46715,1732317251834, state=OPENING 2024-11-22T23:14:15,444 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:14:15,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:15,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:14:15,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:14:15,457 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:14:15,461 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:14:15,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:15,643 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:14:15,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33519, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:14:15,657 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:14:15,657 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:14:15,658 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:14:15,661 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3d87a7361f3%2C46715%2C1732317251834.meta, suffix=.meta, logDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834, archiveDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/oldWALs, maxLogs=32 2024-11-22T23:14:15,678 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834/a3d87a7361f3%2C46715%2C1732317251834.meta.1732317255664.meta, exclude list is [], retry=0 2024-11-22T23:14:15,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41203,DS-97f07e3e-a6bc-4b73-a700-ce5c4605b114,DISK] 2024-11-22T23:14:15,684 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834/a3d87a7361f3%2C46715%2C1732317251834.meta.1732317255664.meta 2024-11-22T23:14:15,685 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36209:36209)] 2024-11-22T23:14:15,685 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:14:15,687 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:14:15,689 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:14:15,694 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:14:15,697 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:14:15,698 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:15,698 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:14:15,698 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:14:15,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T23:14:15,703 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:14:15,703 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,704 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T23:14:15,706 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:14:15,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T23:14:15,708 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:14:15,708 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T23:14:15,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:14:15,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:15,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:14:15,712 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:14:15,714 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740 2024-11-22T23:14:15,716 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740 2024-11-22T23:14:15,719 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:14:15,719 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:14:15,720 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:14:15,722 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:14:15,724 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59389544, jitterRate=-0.11502683162689209}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:14:15,724 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:14:15,726 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732317255699Writing region info on filesystem at 1732317255699Initializing all the Stores at 1732317255701 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317255701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317255701Cleaning up temporary data from old regions at 1732317255719 (+18 ms)Running coprocessor post-open hooks at 1732317255725 (+6 ms)Region opened successfully at 1732317255726 (+1 ms) 2024-11-22T23:14:15,734 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732317255636 2024-11-22T23:14:15,745 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:14:15,745 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:14:15,747 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:15,749 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3d87a7361f3,46715,1732317251834, state=OPEN 2024-11-22T23:14:15,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:14:15,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:14:15,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:14:15,874 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:14:15,875 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:15,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:14:15,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3d87a7361f3,46715,1732317251834 in 411 msec 2024-11-22T23:14:15,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:14:15,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 706 msec 2024-11-22T23:14:15,894 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:14:15,894 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:14:15,915 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:15,917 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:15,933 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:15,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55173, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:15,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7380 sec 2024-11-22T23:14:15,954 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732317255954, completionTime=-1 2024-11-22T23:14:15,957 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T23:14:15,957 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:14:15,983 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:14:15,983 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732317315983 2024-11-22T23:14:15,984 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732317375984 2024-11-22T23:14:15,984 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-22T23:14:15,986 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,986 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,986 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,988 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3d87a7361f3:34971, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,988 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,989 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:15,995 DEBUG [master/a3d87a7361f3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:14:16,030 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.885sec 2024-11-22T23:14:16,032 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:14:16,033 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:14:16,034 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:14:16,034 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:14:16,035 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:14:16,035 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:14:16,036 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:14:16,044 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:14:16,045 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:14:16,045 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,34971,1732317250724-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:14:16,074 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:16,076 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T23:14:16,076 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T23:14:16,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:16,081 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:16,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c924b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:16,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:16,101 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:16,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:16,104 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:16,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:16,114 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:16,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:16,144 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=288, ProcessCount=11, AvailableMemoryMB=1426 2024-11-22T23:14:16,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:14:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-22T23:14:16,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:14:16,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:14:16,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:14:16,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:14:16,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:16,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:14:16,191 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-22T23:14:16,191 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:16,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:14:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:16,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741835_1011 (size=1061) 2024-11-22T23:14:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:16,614 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:14:16,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741836_1012 (size=53) 2024-11-22T23:14:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:17,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:17,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4a71851c051c02efdfd8dcb5ec6b9ff7, disabling compactions & flushes 2024-11-22T23:14:17,025 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,025 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. after waiting 0 ms 2024-11-22T23:14:17,026 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,026 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,026 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: Waiting for close lock at 1732317257025Disabling compacts and flushes for region at 1732317257025Disabling writes for close at 1732317257026 (+1 ms)Writing region close event to WAL at 1732317257026Closed at 1732317257026 2024-11-22T23:14:17,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:14:17,036 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317257029"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317257029"}]},"ts":"1732317257029"} 2024-11-22T23:14:17,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:14:17,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:14:17,045 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317257043"}]},"ts":"1732317257043"} 2024-11-22T23:14:17,050 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:14:17,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, ASSIGN}] 2024-11-22T23:14:17,054 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, ASSIGN 2024-11-22T23:14:17,057 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:14:17,210 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4a71851c051c02efdfd8dcb5ec6b9ff7, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:17,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, ASSIGN because future has completed 2024-11-22T23:14:17,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:17,389 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,389 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:14:17,390 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,390 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:17,390 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,390 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,393 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,397 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:17,397 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a71851c051c02efdfd8dcb5ec6b9ff7 columnFamilyName A 2024-11-22T23:14:17,398 DEBUG [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:17,399 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(327): Store=4a71851c051c02efdfd8dcb5ec6b9ff7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:17,399 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,402 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:17,402 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a71851c051c02efdfd8dcb5ec6b9ff7 columnFamilyName B 2024-11-22T23:14:17,403 DEBUG [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:17,404 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(327): Store=4a71851c051c02efdfd8dcb5ec6b9ff7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:17,404 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,406 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:17,406 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4a71851c051c02efdfd8dcb5ec6b9ff7 columnFamilyName C 2024-11-22T23:14:17,407 DEBUG [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:17,407 INFO [StoreOpener-4a71851c051c02efdfd8dcb5ec6b9ff7-1 {}] regionserver.HStore(327): Store=4a71851c051c02efdfd8dcb5ec6b9ff7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:17,408 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,408 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,410 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,410 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,412 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,412 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,413 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:14:17,415 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,419 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:14:17,420 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4a71851c051c02efdfd8dcb5ec6b9ff7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74845977, jitterRate=0.11529196798801422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:14:17,420 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:17,421 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: Running coprocessor pre-open hook at 1732317257391Writing region info on filesystem at 1732317257391Initializing all the Stores at 1732317257393 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317257393Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317257393Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317257393Cleaning up temporary data from old regions at 1732317257412 (+19 ms)Running coprocessor post-open hooks at 1732317257420 (+8 ms)Region opened successfully at 1732317257421 (+1 ms) 2024-11-22T23:14:17,423 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., pid=6, masterSystemTime=1732317257375 2024-11-22T23:14:17,426 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,426 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:17,427 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4a71851c051c02efdfd8dcb5ec6b9ff7, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:17,431 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:14:17,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:14:17,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 in 215 msec 2024-11-22T23:14:17,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:14:17,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, ASSIGN in 386 msec 2024-11-22T23:14:17,443 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:14:17,443 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317257443"}]},"ts":"1732317257443"} 2024-11-22T23:14:17,447 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:14:17,448 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:14:17,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-22T23:14:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:14:18,350 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:18,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46cd3e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c82fed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,367 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,374 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:14:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@da7d497 2024-11-22T23:14:18,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:14:18,380 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:14:18,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,387 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,390 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,392 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,394 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,394 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68cf4e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,398 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,407 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,412 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7ca9c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,415 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,416 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70343832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,422 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,422 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,422 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,423 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11476af5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,430 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d111af0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,433 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,436 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,437 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e406351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,439 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:18,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:18,442 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17ddb883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:18,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:18,445 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:18,446 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2171b18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:18,450 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,450 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,450 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,450 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,450 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,451 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,451 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,453 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,457 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:18,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:18,461 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:18,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:18,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:18,465 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-22T23:14:18,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:18,477 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:18,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:14:18,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:18,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:18,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:14:18,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:18,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:18,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:18,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:18,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:18,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:18,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:18,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317318666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317258644, max timestamp=1732317258682Num uniques 859; 2024-11-22T23:14:18,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:18,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/c250c15f9a1c44a0b400ac2ae4992284 is 50, key is test_row_0/A:col10/1732317258530/Put/seqid=0 2024-11-22T23:14:18,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317258644, max timestamp=1732317258673Num uniques 859; 2024-11-22T23:14:18,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:18,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317258644, max timestamp=1732317258682Num uniques 859; 2024-11-22T23:14:18,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:18,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741837_1013 (size=12001) 2024-11-22T23:14:18,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/c250c15f9a1c44a0b400ac2ae4992284 2024-11-22T23:14:18,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:18,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317318839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317318848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:18,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:18,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:18,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317318853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/588f935c82d3463bb5a9c5e2adacc5e4 is 50, key is test_row_0/B:col10/1732317258530/Put/seqid=0 2024-11-22T23:14:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317318854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317318869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741838_1014 (size=12001) 2024-11-22T23:14:18,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317318959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,962 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317318959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317318979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317318979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:18,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317319089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:19,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317319169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317319171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317319190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317319190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/588f935c82d3463bb5a9c5e2adacc5e4 2024-11-22T23:14:19,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/f5a2307a616846fe80681a5da8b81d70 is 50, key is test_row_0/C:col10/1732317258530/Put/seqid=0 2024-11-22T23:14:19,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:19,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741839_1015 (size=12001) 2024-11-22T23:14:19,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/f5a2307a616846fe80681a5da8b81d70 2024-11-22T23:14:19,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/c250c15f9a1c44a0b400ac2ae4992284 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284 2024-11-22T23:14:19,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317319398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:14:19,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/588f935c82d3463bb5a9c5e2adacc5e4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4 2024-11-22T23:14:19,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:14:19,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/f5a2307a616846fe80681a5da8b81d70 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70 2024-11-22T23:14:19,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:14:19,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 919ms, sequenceid=12, compaction requested=false 2024-11-22T23:14:19,461 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T23:14:19,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:19,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-22T23:14:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-22T23:14:19,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:19,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:19,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:19,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/ecf911562e3b4b4dadad8b15723e0905 is 50, key is test_row_0/A:col10/1732317258842/Put/seqid=0 2024-11-22T23:14:19,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741840_1016 (size=14341) 2024-11-22T23:14:19,545 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/ecf911562e3b4b4dadad8b15723e0905 2024-11-22T23:14:19,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/9a09347220bb4f22956b7a6246e302e5 is 50, key is test_row_0/B:col10/1732317258842/Put/seqid=0 2024-11-22T23:14:19,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741841_1017 (size=14341) 2024-11-22T23:14:19,601 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/9a09347220bb4f22956b7a6246e302e5 2024-11-22T23:14:19,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317319580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317319592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317319598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:19,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317319606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:19,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5fd01d2b59fa4f2f88a6ee38e383c452 is 50, key is test_row_0/C:col10/1732317258842/Put/seqid=0 2024-11-22T23:14:19,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741842_1018 (size=12001) 2024-11-22T23:14:19,653 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5fd01d2b59fa4f2f88a6ee38e383c452 2024-11-22T23:14:19,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/ecf911562e3b4b4dadad8b15723e0905 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905 2024-11-22T23:14:19,682 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905, entries=200, sequenceid=49, filesize=14.0 K 2024-11-22T23:14:19,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/9a09347220bb4f22956b7a6246e302e5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5 2024-11-22T23:14:19,710 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5, entries=200, sequenceid=49, filesize=14.0 K 2024-11-22T23:14:19,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5fd01d2b59fa4f2f88a6ee38e383c452 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452 2024-11-22T23:14:19,727 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452, entries=150, sequenceid=49, filesize=11.7 K 2024-11-22T23:14:19,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 231ms, sequenceid=49, compaction requested=false 2024-11-22T23:14:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-22T23:14:19,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-22T23:14:19,738 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T23:14:19,739 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2500 sec 2024-11-22T23:14:19,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2790 sec 2024-11-22T23:14:19,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:19,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:19,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:19,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/fe64bc678e524dd39b50492fd229e786 is 50, key is test_row_0/A:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:19,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741843_1019 (size=14341) 2024-11-22T23:14:19,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/fe64bc678e524dd39b50492fd229e786 2024-11-22T23:14:19,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/e02e2f124b1c46dfa9353dbe96358ebb is 50, key is test_row_0/B:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:19,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741844_1020 (size=12001) 2024-11-22T23:14:19,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/e02e2f124b1c46dfa9353dbe96358ebb 2024-11-22T23:14:19,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eba02b0876cb4f2fb8a55f0284506b5c is 50, key is test_row_0/C:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741845_1021 (size=12001) 2024-11-22T23:14:19,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eba02b0876cb4f2fb8a55f0284506b5c 2024-11-22T23:14:19,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/fe64bc678e524dd39b50492fd229e786 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786 2024-11-22T23:14:19,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786, entries=200, sequenceid=60, filesize=14.0 K 2024-11-22T23:14:19,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/e02e2f124b1c46dfa9353dbe96358ebb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb 2024-11-22T23:14:19,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:14:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eba02b0876cb4f2fb8a55f0284506b5c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c 2024-11-22T23:14:19,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-22T23:14:19,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 207ms, sequenceid=60, compaction requested=true 2024-11-22T23:14:19,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:19,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:19,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:19,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:19,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-22T23:14:19,977 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:19,977 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:19,981 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:19,982 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:19,983 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:19,983 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:19,984 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,984 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:19,984 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=37.4 K 2024-11-22T23:14:19,984 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=39.7 K 2024-11-22T23:14:19,986 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 588f935c82d3463bb5a9c5e2adacc5e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317258516 2024-11-22T23:14:19,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c250c15f9a1c44a0b400ac2ae4992284, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317258516 2024-11-22T23:14:19,988 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a09347220bb4f22956b7a6246e302e5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732317258842 2024-11-22T23:14:19,988 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecf911562e3b4b4dadad8b15723e0905, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732317258842 2024-11-22T23:14:19,990 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e02e2f124b1c46dfa9353dbe96358ebb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:19,990 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe64bc678e524dd39b50492fd229e786, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:19,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317259819, max timestamp=1732317259973Num uniques 859; 2024-11-22T23:14:19,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:14:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:20,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:20,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:20,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/0904ec0a45954a22bc486bd57773ef68 is 50, key is test_row_0/A:col10/1732317259819/Put/seqid=0 2024-11-22T23:14:20,061 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#11 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,061 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#9 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,062 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/480a90870213465b93a28481ff1a5705 is 50, key is test_row_0/B:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:20,074 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1f55234e617c408f908f0baa630afa45 is 50, key is test_row_0/A:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:20,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741847_1023 (size=12104) 2024-11-22T23:14:20,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741848_1024 (size=12104) 2024-11-22T23:14:20,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741846_1022 (size=19021) 2024-11-22T23:14:20,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/0904ec0a45954a22bc486bd57773ef68 2024-11-22T23:14:20,107 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1f55234e617c408f908f0baa630afa45 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f55234e617c408f908f0baa630afa45 2024-11-22T23:14:20,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5b13e782be946deb5600f3b15b844a0 is 50, key is test_row_0/B:col10/1732317259819/Put/seqid=0 2024-11-22T23:14:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317320101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317320116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317320123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,134 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,143 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 1f55234e617c408f908f0baa630afa45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:20,143 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317320131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,146 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317259972; duration=0sec 2024-11-22T23:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,147 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,147 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:20,147 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:20,152 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:20,152 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:20,152 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:20,153 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.2 K 2024-11-22T23:14:20,154 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a2307a616846fe80681a5da8b81d70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317258516 2024-11-22T23:14:20,155 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fd01d2b59fa4f2f88a6ee38e383c452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732317258842 2024-11-22T23:14:20,156 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting eba02b0876cb4f2fb8a55f0284506b5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:20,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317320133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741849_1025 (size=12001) 2024-11-22T23:14:20,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5b13e782be946deb5600f3b15b844a0 2024-11-22T23:14:20,193 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#13 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,194 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b28091fff33b4e42a7e69746c0e5501c is 50, key is test_row_0/C:col10/1732317259572/Put/seqid=0 2024-11-22T23:14:20,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e79024e739f5472f8105bb9b35ae9709 is 50, key is test_row_0/C:col10/1732317259819/Put/seqid=0 2024-11-22T23:14:20,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741850_1026 (size=12104) 2024-11-22T23:14:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317320238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317320238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317320239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,248 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b28091fff33b4e42a7e69746c0e5501c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b28091fff33b4e42a7e69746c0e5501c 2024-11-22T23:14:20,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741851_1027 (size=12001) 2024-11-22T23:14:20,253 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e79024e739f5472f8105bb9b35ae9709 2024-11-22T23:14:20,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317320259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/0904ec0a45954a22bc486bd57773ef68 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68 2024-11-22T23:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317320271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,281 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into b28091fff33b4e42a7e69746c0e5501c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:20,281 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,282 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317259976; duration=0sec 2024-11-22T23:14:20,283 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:20,283 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:20,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68, entries=300, sequenceid=81, filesize=18.6 K 2024-11-22T23:14:20,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5b13e782be946deb5600f3b15b844a0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0 2024-11-22T23:14:20,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0, entries=150, sequenceid=81, filesize=11.7 K 2024-11-22T23:14:20,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e79024e739f5472f8105bb9b35ae9709 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709 2024-11-22T23:14:20,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709, entries=150, sequenceid=81, filesize=11.7 K 2024-11-22T23:14:20,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~272.22 KB/278752, currentSize=87.22 KB/89310 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 355ms, sequenceid=81, compaction requested=false 2024-11-22T23:14:20,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,411 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T23:14:20,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:20,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:20,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/d3f8771ca59442ea8d6525943d5d9e68 is 50, key is test_row_0/A:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,499 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/480a90870213465b93a28481ff1a5705 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/480a90870213465b93a28481ff1a5705 2024-11-22T23:14:20,521 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 480a90870213465b93a28481ff1a5705(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:20,521 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,521 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317259976; duration=0sec 2024-11-22T23:14:20,522 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:20,522 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:20,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741852_1028 (size=16681) 2024-11-22T23:14:20,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317320509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/d3f8771ca59442ea8d6525943d5d9e68 2024-11-22T23:14:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317320520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317320523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b1d00a0cf9fb4a9ea0a155b20bd801eb is 50, key is test_row_0/B:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317320526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317320529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741853_1029 (size=12001) 2024-11-22T23:14:20,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b1d00a0cf9fb4a9ea0a155b20bd801eb 2024-11-22T23:14:20,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/87ea5b57c31f475e8897687fe4ca6d9b is 50, key is test_row_0/C:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T23:14:20,639 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:20,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:20,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-22T23:14:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:14:20,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317320638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,646 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,648 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:20,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:20,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741854_1030 (size=12001) 2024-11-22T23:14:20,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/87ea5b57c31f475e8897687fe4ca6d9b 2024-11-22T23:14:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317320648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317320653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/d3f8771ca59442ea8d6525943d5d9e68 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68 2024-11-22T23:14:20,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317320658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317320658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68, entries=250, sequenceid=101, filesize=16.3 K 2024-11-22T23:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b1d00a0cf9fb4a9ea0a155b20bd801eb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb 2024-11-22T23:14:20,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb, entries=150, sequenceid=101, filesize=11.7 K 2024-11-22T23:14:20,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/87ea5b57c31f475e8897687fe4ca6d9b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b 2024-11-22T23:14:20,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b, entries=150, sequenceid=101, filesize=11.7 K 2024-11-22T23:14:20,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 272ms, sequenceid=101, compaction requested=true 2024-11-22T23:14:20,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:20,730 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:20,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:20,731 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:20,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:20,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:20,734 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:20,734 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47806 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:20,734 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:20,734 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:20,734 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:20,734 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:20,734 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/480a90870213465b93a28481ff1a5705, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.3 K 2024-11-22T23:14:20,735 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f55234e617c408f908f0baa630afa45, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=46.7 K 2024-11-22T23:14:20,735 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 480a90870213465b93a28481ff1a5705, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:20,736 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f55234e617c408f908f0baa630afa45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:20,736 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b13e782be946deb5600f3b15b844a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732317259819 2024-11-22T23:14:20,736 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0904ec0a45954a22bc486bd57773ef68, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732317259819 2024-11-22T23:14:20,737 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b1d00a0cf9fb4a9ea0a155b20bd801eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:20,737 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f8771ca59442ea8d6525943d5d9e68, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:20,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:14:20,769 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,770 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/32399f2beb1f4289b0e6f5e6fc613c74 is 50, key is test_row_0/A:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,779 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,780 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/6c6abb015aa4449a94d1435d6582b44e is 50, key is test_row_0/B:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-22T23:14:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:20,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741855_1031 (size=12207) 2024-11-22T23:14:20,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741856_1032 (size=12207) 2024-11-22T23:14:20,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a3ff9a3dcf2e432eb66c4a7a1bace89c is 50, key is test_row_0/A:col10/1732317260519/Put/seqid=0 2024-11-22T23:14:20,843 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/6c6abb015aa4449a94d1435d6582b44e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/6c6abb015aa4449a94d1435d6582b44e 2024-11-22T23:14:20,856 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 6c6abb015aa4449a94d1435d6582b44e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:20,858 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,858 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317260731; duration=0sec 2024-11-22T23:14:20,858 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:20,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:20,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:20,864 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:20,864 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:20,864 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:20,865 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b28091fff33b4e42a7e69746c0e5501c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.3 K 2024-11-22T23:14:20,866 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b28091fff33b4e42a7e69746c0e5501c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1732317259572 2024-11-22T23:14:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:20,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:20,872 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e79024e739f5472f8105bb9b35ae9709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732317259819 2024-11-22T23:14:20,874 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ea5b57c31f475e8897687fe4ca6d9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:20,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741857_1033 (size=12001) 2024-11-22T23:14:20,887 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a3ff9a3dcf2e432eb66c4a7a1bace89c 2024-11-22T23:14:20,909 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:20,910 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/932c00f6b740419aa9deb3179038048b is 50, key is test_row_0/C:col10/1732317260088/Put/seqid=0 2024-11-22T23:14:20,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5d84840976e4226b6bbbaefb3da40fb is 50, key is test_row_0/B:col10/1732317260519/Put/seqid=0 2024-11-22T23:14:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317320905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317320921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317320923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317320929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317320930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:14:20,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741858_1034 (size=12207) 2024-11-22T23:14:20,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741859_1035 (size=12001) 2024-11-22T23:14:20,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5d84840976e4226b6bbbaefb3da40fb 2024-11-22T23:14:20,980 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/932c00f6b740419aa9deb3179038048b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/932c00f6b740419aa9deb3179038048b 2024-11-22T23:14:20,990 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 932c00f6b740419aa9deb3179038048b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:20,990 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:20,990 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317260733; duration=0sec 2024-11-22T23:14:20,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/22f9655d2996432db7a7f8d364192f4e is 50, key is test_row_0/C:col10/1732317260519/Put/seqid=0 2024-11-22T23:14:20,991 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:20,991 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:21,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741860_1036 (size=12001) 2024-11-22T23:14:21,021 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/22f9655d2996432db7a7f8d364192f4e 2024-11-22T23:14:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a3ff9a3dcf2e432eb66c4a7a1bace89c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c 2024-11-22T23:14:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317321038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317321039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317321048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317321048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317321048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,054 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T23:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d5d84840976e4226b6bbbaefb3da40fb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb 2024-11-22T23:14:21,074 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T23:14:21,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/22f9655d2996432db7a7f8d364192f4e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e 2024-11-22T23:14:21,086 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-22T23:14:21,088 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 283ms, sequenceid=117, compaction requested=false 2024-11-22T23:14:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-22T23:14:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-22T23:14:21,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-22T23:14:21,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 444 msec 2024-11-22T23:14:21,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 458 msec 2024-11-22T23:14:21,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:14:21,247 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/32399f2beb1f4289b0e6f5e6fc613c74 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/32399f2beb1f4289b0e6f5e6fc613c74 2024-11-22T23:14:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:21,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:14:21,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:21,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-22T23:14:21,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:21,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-22T23:14:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:21,272 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:21,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:21,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:21,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/8c7272603c1e4449bee697d47233cbbc is 50, key is test_row_0/A:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:21,289 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 32399f2beb1f4289b0e6f5e6fc613c74(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:21,289 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:21,289 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317260730; duration=0sec 2024-11-22T23:14:21,290 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:21,290 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:21,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317321284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741861_1037 (size=14541) 2024-11-22T23:14:21,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/8c7272603c1e4449bee697d47233cbbc 2024-11-22T23:14:21,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317321295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,316 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T23:14:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317321298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,316 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T23:14:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:14:21,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T23:14:21,319 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T23:14:21,319 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T23:14:21,320 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:14:21,321 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:14:21,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317321306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317321292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/f61a051d75bd4afe84ac66bbfcb98bae is 50, key is test_row_0/B:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:21,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741862_1038 (size=12151) 2024-11-22T23:14:21,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/f61a051d75bd4afe84ac66bbfcb98bae 2024-11-22T23:14:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:21,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e3f3f1ca7e10480191b405f27fa2648e is 50, key is test_row_0/C:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:21,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741863_1039 (size=12151) 2024-11-22T23:14:21,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317321408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317321417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:14:21,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317321429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317321429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317321430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:14:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:21,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317321617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317321627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317321638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317321639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317321638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:21,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:14:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:21,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e3f3f1ca7e10480191b405f27fa2648e 2024-11-22T23:14:21,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/8c7272603c1e4449bee697d47233cbbc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc 2024-11-22T23:14:21,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc, entries=200, sequenceid=139, filesize=14.2 K 2024-11-22T23:14:21,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/f61a051d75bd4afe84ac66bbfcb98bae as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae 2024-11-22T23:14:21,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T23:14:21,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e3f3f1ca7e10480191b405f27fa2648e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e 2024-11-22T23:14:21,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T23:14:21,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 628ms, sequenceid=139, compaction requested=true 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:21,882 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:21,882 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:21,886 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:21,886 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:21,886 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,886 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/6c6abb015aa4449a94d1435d6582b44e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.5 K 2024-11-22T23:14:21,887 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c6abb015aa4449a94d1435d6582b44e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:21,888 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d5d84840976e4226b6bbbaefb3da40fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732317260512 2024-11-22T23:14:21,888 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:21,889 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:21,889 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,889 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/32399f2beb1f4289b0e6f5e6fc613c74, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=37.8 K 2024-11-22T23:14:21,890 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32399f2beb1f4289b0e6f5e6fc613c74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:21,890 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f61a051d75bd4afe84ac66bbfcb98bae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260924 2024-11-22T23:14:21,891 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ff9a3dcf2e432eb66c4a7a1bace89c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732317260512 2024-11-22T23:14:21,892 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c7272603c1e4449bee697d47233cbbc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260922 2024-11-22T23:14:21,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-22T23:14:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:21,897 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:14:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:21,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/90f2cd6ddf694a5e8d952b9c0cfd8181 is 50, key is test_row_0/A:col10/1732317261268/Put/seqid=0 2024-11-22T23:14:21,920 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#28 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:21,920 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:21,925 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8d04d0666d9b44dd90f0ec0af600d553 is 50, key is test_row_0/B:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:21,931 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b914de47580b4b89829d39eaca8e1775 is 50, key is test_row_0/A:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:21,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741864_1040 (size=12151) 2024-11-22T23:14:21,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741865_1041 (size=12459) 2024-11-22T23:14:21,969 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8d04d0666d9b44dd90f0ec0af600d553 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8d04d0666d9b44dd90f0ec0af600d553 2024-11-22T23:14:21,990 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 8d04d0666d9b44dd90f0ec0af600d553(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:21,990 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:21,990 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317261882; duration=0sec 2024-11-22T23:14:21,990 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:21,990 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:21,991 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:21,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741866_1042 (size=12459) 2024-11-22T23:14:22,003 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:22,003 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:22,004 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:22,005 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/932c00f6b740419aa9deb3179038048b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.5 K 2024-11-22T23:14:22,006 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 932c00f6b740419aa9deb3179038048b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732317260088 2024-11-22T23:14:22,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f9655d2996432db7a7f8d364192f4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732317260512 2024-11-22T23:14:22,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317321995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317321998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317322006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317322010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317322013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,023 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e3f3f1ca7e10480191b405f27fa2648e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260924 2024-11-22T23:14:22,051 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:22,053 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/fb4d57bdf3824ce5b8dc9bcd183e3774 is 50, key is test_row_0/C:col10/1732317261249/Put/seqid=0 2024-11-22T23:14:22,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741867_1043 (size=12459) 2024-11-22T23:14:22,086 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/fb4d57bdf3824ce5b8dc9bcd183e3774 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/fb4d57bdf3824ce5b8dc9bcd183e3774 2024-11-22T23:14:22,098 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into fb4d57bdf3824ce5b8dc9bcd183e3774(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:22,098 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:22,099 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317261882; duration=0sec 2024-11-22T23:14:22,099 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:22,099 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317322117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317322117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317322129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317322129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317322131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317322328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317322328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317322337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317322337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317322337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,345 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/90f2cd6ddf694a5e8d952b9c0cfd8181 2024-11-22T23:14:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/497c212f0ce6497fa922f57465c40ea9 is 50, key is test_row_0/B:col10/1732317261268/Put/seqid=0 2024-11-22T23:14:22,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741868_1044 (size=12151) 2024-11-22T23:14:22,380 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/497c212f0ce6497fa922f57465c40ea9 2024-11-22T23:14:22,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/69f7db8059534927a91e45e47babf5b3 is 50, key is test_row_0/C:col10/1732317261268/Put/seqid=0 2024-11-22T23:14:22,405 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b914de47580b4b89829d39eaca8e1775 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b914de47580b4b89829d39eaca8e1775 2024-11-22T23:14:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:22,416 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into b914de47580b4b89829d39eaca8e1775(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:22,416 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:22,416 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317261882; duration=0sec 2024-11-22T23:14:22,416 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:22,416 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:22,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741869_1045 (size=12151) 2024-11-22T23:14:22,428 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/69f7db8059534927a91e45e47babf5b3 2024-11-22T23:14:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/90f2cd6ddf694a5e8d952b9c0cfd8181 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181 2024-11-22T23:14:22,452 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181, entries=150, sequenceid=153, filesize=11.9 K 2024-11-22T23:14:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/497c212f0ce6497fa922f57465c40ea9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9 2024-11-22T23:14:22,466 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9, entries=150, sequenceid=153, filesize=11.9 K 2024-11-22T23:14:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/69f7db8059534927a91e45e47babf5b3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3 2024-11-22T23:14:22,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3, entries=150, sequenceid=153, filesize=11.9 K 2024-11-22T23:14:22,487 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 590ms, sequenceid=153, compaction requested=false 2024-11-22T23:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-22T23:14:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-22T23:14:22,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T23:14:22,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-22T23:14:22,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.2290 sec 2024-11-22T23:14:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:22,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-22T23:14:22,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732317261987, max timestamp=1732317262641Num uniques 859; 2024-11-22T23:14:22,649 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:22,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:22,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:22,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:22,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317261987, max timestamp=1732317262641Num uniques 859; 2024-11-22T23:14:22,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:22,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:22,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:22,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:22,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/3260f3f5a2564392b1349e64677a1e19 is 50, key is test_row_0/A:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:22,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741870_1046 (size=14541) 2024-11-22T23:14:22,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317262736, max timestamp=1732317262756Num uniques 859; 2024-11-22T23:14:22,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317262736, max timestamp=1732317262756Num uniques 859; 2024-11-22T23:14:22,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317262736, max timestamp=1732317262756Num uniques 859; 2024-11-22T23:14:22,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:22,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:22,762 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:22,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317322780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317322785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317322787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317322792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317322792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317322898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317322898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317322908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317322927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317322927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:22,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/3260f3f5a2564392b1349e64677a1e19 2024-11-22T23:14:23,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 is 50, key is test_row_0/B:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:23,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741871_1047 (size=12151) 2024-11-22T23:14:23,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 2024-11-22T23:14:23,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317323107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317323107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e8aabbd1ad674b9aa82c7782da19275c is 50, key is test_row_0/C:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317323118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741872_1048 (size=12151) 2024-11-22T23:14:23,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e8aabbd1ad674b9aa82c7782da19275c 2024-11-22T23:14:23,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317323138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/3260f3f5a2564392b1349e64677a1e19 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19 2024-11-22T23:14:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317323139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19, entries=200, sequenceid=177, filesize=14.2 K 2024-11-22T23:14:23,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 2024-11-22T23:14:23,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T23:14:23,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/e8aabbd1ad674b9aa82c7782da19275c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c 2024-11-22T23:14:23,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T23:14:23,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~169.20 KB/173264, currentSize=127.47 KB/130530 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 541ms, sequenceid=177, compaction requested=true 2024-11-22T23:14:23,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:23,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:23,186 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:23,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:23,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:23,187 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:23,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:23,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:23,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:23,188 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:23,188 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:23,188 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:23,188 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:23,188 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,188 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,188 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8d04d0666d9b44dd90f0ec0af600d553, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.9 K 2024-11-22T23:14:23,188 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b914de47580b4b89829d39eaca8e1775, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=38.2 K 2024-11-22T23:14:23,192 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d04d0666d9b44dd90f0ec0af600d553, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260924 2024-11-22T23:14:23,193 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b914de47580b4b89829d39eaca8e1775, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260924 2024-11-22T23:14:23,194 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 497c212f0ce6497fa922f57465c40ea9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732317261268 2024-11-22T23:14:23,194 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f2cd6ddf694a5e8d952b9c0cfd8181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732317261268 2024-11-22T23:14:23,194 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3260f3f5a2564392b1349e64677a1e19, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317261987 2024-11-22T23:14:23,195 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b6b27aac7a7a4cbdb7c92c83c64b5a87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317262002 2024-11-22T23:14:23,208 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:23,208 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/58450bb1987642ca9b8d12c9428c68a2 is 50, key is test_row_0/B:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:23,215 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:23,215 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/fa97689d71e34068b771b2e6ce1fc1be is 50, key is test_row_0/A:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:23,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741873_1049 (size=12561) 2024-11-22T23:14:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741874_1050 (size=12561) 2024-11-22T23:14:23,259 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/58450bb1987642ca9b8d12c9428c68a2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/58450bb1987642ca9b8d12c9428c68a2 2024-11-22T23:14:23,260 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/fa97689d71e34068b771b2e6ce1fc1be as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fa97689d71e34068b771b2e6ce1fc1be 2024-11-22T23:14:23,275 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 58450bb1987642ca9b8d12c9428c68a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:23,275 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:23,276 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317263187; duration=0sec 2024-11-22T23:14:23,276 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:23,276 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:23,276 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:23,276 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into fa97689d71e34068b771b2e6ce1fc1be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:23,277 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:23,277 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317263186; duration=0sec 2024-11-22T23:14:23,277 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:23,277 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:23,279 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:23,279 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:23,279 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,280 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/fb4d57bdf3824ce5b8dc9bcd183e3774, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=35.9 K 2024-11-22T23:14:23,280 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fb4d57bdf3824ce5b8dc9bcd183e3774, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317260924 2024-11-22T23:14:23,281 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f7db8059534927a91e45e47babf5b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732317261268 2024-11-22T23:14:23,281 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e8aabbd1ad674b9aa82c7782da19275c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317262002 2024-11-22T23:14:23,307 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#38 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:23,307 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/83be63cb77804ad7b73c1634d3be52f1 is 50, key is test_row_0/C:col10/1732317262002/Put/seqid=0 2024-11-22T23:14:23,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741875_1051 (size=12561) 2024-11-22T23:14:23,345 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/83be63cb77804ad7b73c1634d3be52f1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/83be63cb77804ad7b73c1634d3be52f1 2024-11-22T23:14:23,358 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 83be63cb77804ad7b73c1634d3be52f1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:23,358 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:23,358 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317263187; duration=0sec 2024-11-22T23:14:23,358 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:23,358 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-22T23:14:23,418 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:23,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:23,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-22T23:14:23,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:23,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-22T23:14:23,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:23,425 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:23,427 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:23,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:23,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:23,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:23,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9c8290de913947a988baa48efa63734c is 50, key is test_row_0/A:col10/1732317263418/Put/seqid=0 2024-11-22T23:14:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317323449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317323455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741876_1052 (size=14541) 2024-11-22T23:14:23,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317323458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317323459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317323460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:23,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317323568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317323568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317323578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:23,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317323778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317323778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317323788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9c8290de913947a988baa48efa63734c 2024-11-22T23:14:23,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/831316dac444466db7bb687ea71a4a36 is 50, key is test_row_0/B:col10/1732317263418/Put/seqid=0 2024-11-22T23:14:23,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:23,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:23,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:23,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741877_1053 (size=12151) 2024-11-22T23:14:23,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/831316dac444466db7bb687ea71a4a36 2024-11-22T23:14:23,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/a69c573532e940209ff271bfc3728585 is 50, key is test_row_0/C:col10/1732317263418/Put/seqid=0 2024-11-22T23:14:23,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741878_1054 (size=12151) 2024-11-22T23:14:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:24,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317324088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317324088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317324097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/a69c573532e940209ff271bfc3728585 2024-11-22T23:14:24,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9c8290de913947a988baa48efa63734c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c 2024-11-22T23:14:24,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c, entries=200, sequenceid=214, filesize=14.2 K 2024-11-22T23:14:24,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/831316dac444466db7bb687ea71a4a36 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36 2024-11-22T23:14:24,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:24,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:24,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36, entries=150, sequenceid=214, filesize=11.9 K 2024-11-22T23:14:24,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:24,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/a69c573532e940209ff271bfc3728585 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585 2024-11-22T23:14:24,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585, entries=150, sequenceid=214, filesize=11.9 K 2024-11-22T23:14:24,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 965ms, sequenceid=214, compaction requested=false 2024-11-22T23:14:24,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:24,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-22T23:14:24,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:24,523 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:24,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/06c6baa5db024f7dbe8508e5ee23686b is 50, key is test_row_0/A:col10/1732317263443/Put/seqid=0 2024-11-22T23:14:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:24,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741879_1055 (size=12151) 2024-11-22T23:14:24,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:24,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317264674, max timestamp=1732317264746Num uniques 859; 2024-11-22T23:14:24,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:24,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317264674, max timestamp=1732317264747Num uniques 859; 2024-11-22T23:14:24,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317264674, max timestamp=1732317264746Num uniques 859; 2024-11-22T23:14:24,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:24,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317324809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317324813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317324824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317324928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317324928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317324947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:24,981 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/06c6baa5db024f7dbe8508e5ee23686b 2024-11-22T23:14:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/ac35c3b373e244d2b644ebe70ef9c6b7 is 50, key is test_row_0/B:col10/1732317263443/Put/seqid=0 2024-11-22T23:14:24,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317324989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:24,998 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317324998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,000 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741880_1056 (size=12151) 2024-11-22T23:14:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317325138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317325138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317325157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,404 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/ac35c3b373e244d2b644ebe70ef9c6b7 2024-11-22T23:14:25,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f0397bafd214145ab59d0fa3afee5fe is 50, key is test_row_0/C:col10/1732317263443/Put/seqid=0 2024-11-22T23:14:25,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741881_1057 (size=12151) 2024-11-22T23:14:25,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317325448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317325448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,453 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f0397bafd214145ab59d0fa3afee5fe 2024-11-22T23:14:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,454 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/06c6baa5db024f7dbe8508e5ee23686b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b 2024-11-22T23:14:25,472 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b, entries=150, sequenceid=224, filesize=11.9 K 2024-11-22T23:14:25,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317325468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/ac35c3b373e244d2b644ebe70ef9c6b7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7 2024-11-22T23:14:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,503 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7, entries=150, sequenceid=224, filesize=11.9 K 2024-11-22T23:14:25,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f0397bafd214145ab59d0fa3afee5fe as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe 2024-11-22T23:14:25,518 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe, entries=150, sequenceid=224, filesize=11.9 K 2024-11-22T23:14:25,520 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 998ms, sequenceid=224, compaction requested=true 2024-11-22T23:14:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:25,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-22T23:14:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-22T23:14:25,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T23:14:25,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1010 sec 2024-11-22T23:14:25,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1150 sec 2024-11-22T23:14:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T23:14:25,568 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:25,569 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-22T23:14:25,573 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:14:25,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:25,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:14:25,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-22T23:14:25,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:25,730 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:25,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b2e98bedd78146e8ba27219bbe43b076 is 50, key is test_row_0/A:col10/1732317264804/Put/seqid=0 2024-11-22T23:14:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741882_1058 (size=12301) 2024-11-22T23:14:25,751 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b2e98bedd78146e8ba27219bbe43b076 2024-11-22T23:14:25,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d14d292b189745798ecdce3c7f47b581 is 50, key is test_row_0/B:col10/1732317264804/Put/seqid=0 2024-11-22T23:14:25,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741883_1059 (size=12301) 2024-11-22T23:14:25,792 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d14d292b189745798ecdce3c7f47b581 2024-11-22T23:14:25,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f49a7191d1046d4ae98469888b362a6 is 50, key is test_row_0/C:col10/1732317264804/Put/seqid=0 2024-11-22T23:14:25,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741884_1060 (size=12301) 2024-11-22T23:14:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:14:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:25,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317325983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317325984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:25,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317325993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317326097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317326098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317326108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:14:26,269 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f49a7191d1046d4ae98469888b362a6 2024-11-22T23:14:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b2e98bedd78146e8ba27219bbe43b076 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076 2024-11-22T23:14:26,285 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:14:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d14d292b189745798ecdce3c7f47b581 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581 2024-11-22T23:14:26,294 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:14:26,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/5f49a7191d1046d4ae98469888b362a6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6 2024-11-22T23:14:26,304 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6, entries=150, sequenceid=260, filesize=12.0 K 2024-11-22T23:14:26,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 575ms, sequenceid=260, compaction requested=true 2024-11-22T23:14:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:26,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-22T23:14:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-22T23:14:26,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:26,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-22T23:14:26,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 734 msec 2024-11-22T23:14:26,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 747 msec 2024-11-22T23:14:26,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5c96f1a3a0f84c579871c754f6969dee is 50, key is test_row_0/A:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741885_1061 (size=14741) 2024-11-22T23:14:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317326381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317326383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,391 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317266341, max timestamp=1732317266387Num uniques 859; 2024-11-22T23:14:26,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317266341, max timestamp=1732317266387Num uniques 859; 2024-11-22T23:14:26,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317266341, max timestamp=1732317266387Num uniques 859; 2024-11-22T23:14:26,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:26,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:26,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317326487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317326497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317326497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317326597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-22T23:14:26,708 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:26,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-22T23:14:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:14:26,710 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:26,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317326708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,711 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:26,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317326717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5c96f1a3a0f84c579871c754f6969dee 2024-11-22T23:14:26,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d2aa13b6990e451ebd0429a1ba38917a is 50, key is test_row_0/B:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:26,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741886_1062 (size=12301) 2024-11-22T23:14:26,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317326808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:14:26,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:26,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:26,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:26,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:26,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:26,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317327018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317327018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,024 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317327018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,025 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:14:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317327027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317327127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,130 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d2aa13b6990e451ebd0429a1ba38917a 2024-11-22T23:14:27,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/ea0b03eaa3254a07a7cce5424d73bfe7 is 50, key is test_row_0/C:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:27,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:27,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741887_1063 (size=12301) 2024-11-22T23:14:27,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:27,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:14:27,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:27,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:27,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:27,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317327538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317327538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/ea0b03eaa3254a07a7cce5424d73bfe7 2024-11-22T23:14:27,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5c96f1a3a0f84c579871c754f6969dee as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee 2024-11-22T23:14:27,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee, entries=200, sequenceid=271, filesize=14.4 K 2024-11-22T23:14:27,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d2aa13b6990e451ebd0429a1ba38917a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a 2024-11-22T23:14:27,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T23:14:27,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/ea0b03eaa3254a07a7cce5424d73bfe7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7 2024-11-22T23:14:27,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7, entries=150, sequenceid=271, filesize=12.0 K 2024-11-22T23:14:27,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 1299ms, sequenceid=271, compaction requested=true 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:27,610 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:27,610 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:27,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:27,612 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 66295 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:14:27,612 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:27,612 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,612 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fa97689d71e34068b771b2e6ce1fc1be, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=64.7 K 2024-11-22T23:14:27,613 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:14:27,613 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:27,613 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,613 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa97689d71e34068b771b2e6ce1fc1be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317262002 2024-11-22T23:14:27,613 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/58450bb1987642ca9b8d12c9428c68a2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=60.0 K 2024-11-22T23:14:27,613 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 58450bb1987642ca9b8d12c9428c68a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317262002 2024-11-22T23:14:27,613 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c8290de913947a988baa48efa63734c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732317262784 2024-11-22T23:14:27,614 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 831316dac444466db7bb687ea71a4a36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732317262787 2024-11-22T23:14:27,614 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06c6baa5db024f7dbe8508e5ee23686b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732317263443 2024-11-22T23:14:27,614 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ac35c3b373e244d2b644ebe70ef9c6b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732317263443 2024-11-22T23:14:27,614 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e98bedd78146e8ba27219bbe43b076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732317264804 2024-11-22T23:14:27,615 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d14d292b189745798ecdce3c7f47b581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732317264804 2024-11-22T23:14:27,615 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c96f1a3a0f84c579871c754f6969dee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:27,616 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d2aa13b6990e451ebd0429a1ba38917a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:27,631 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:27,631 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:27,632 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/2368e7ae652a4a24afa56147bd3abcf1 is 50, key is test_row_0/A:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:27,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-22T23:14:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,636 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:27,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:27,637 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/e5451439d1244d879352df06880a4171 is 50, key is test_row_0/B:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:27,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741888_1064 (size=12881) 2024-11-22T23:14:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:27,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:27,651 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/2368e7ae652a4a24afa56147bd3abcf1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/2368e7ae652a4a24afa56147bd3abcf1 2024-11-22T23:14:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6731b7bb10a84e70b30eb2dedcfc0bbc is 50, key is test_row_0/A:col10/1732317266485/Put/seqid=0 2024-11-22T23:14:27,659 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 2368e7ae652a4a24afa56147bd3abcf1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:27,659 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:27,660 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=11, startTime=1732317267610; duration=0sec 2024-11-22T23:14:27,661 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:27,661 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:27,661 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-22T23:14:27,663 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-22T23:14:27,663 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:27,663 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,664 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/83be63cb77804ad7b73c1634d3be52f1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=60.0 K 2024-11-22T23:14:27,664 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83be63cb77804ad7b73c1634d3be52f1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317262002 2024-11-22T23:14:27,665 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a69c573532e940209ff271bfc3728585, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732317262787 2024-11-22T23:14:27,665 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0397bafd214145ab59d0fa3afee5fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1732317263443 2024-11-22T23:14:27,665 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f49a7191d1046d4ae98469888b362a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1732317264804 2024-11-22T23:14:27,666 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea0b03eaa3254a07a7cce5424d73bfe7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:27,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741889_1065 (size=12881) 2024-11-22T23:14:27,693 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/e5451439d1244d879352df06880a4171 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e5451439d1244d879352df06880a4171 2024-11-22T23:14:27,694 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#54 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:27,695 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/62b21b9f5e974173a343f47814dc196f is 50, key is test_row_0/C:col10/1732317265972/Put/seqid=0 2024-11-22T23:14:27,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741890_1066 (size=12301) 2024-11-22T23:14:27,698 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6731b7bb10a84e70b30eb2dedcfc0bbc 2024-11-22T23:14:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317327701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,707 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into e5451439d1244d879352df06880a4171(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:27,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:27,707 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=11, startTime=1732317267610; duration=0sec 2024-11-22T23:14:27,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:27,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:27,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741891_1067 (size=12881) 2024-11-22T23:14:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/edc9229dae8f49738b8b8748442f8371 is 50, key is test_row_0/B:col10/1732317266485/Put/seqid=0 2024-11-22T23:14:27,717 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/62b21b9f5e974173a343f47814dc196f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/62b21b9f5e974173a343f47814dc196f 2024-11-22T23:14:27,723 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 62b21b9f5e974173a343f47814dc196f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:27,723 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:27,723 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=11, startTime=1732317267610; duration=0sec 2024-11-22T23:14:27,724 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:27,724 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:27,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741892_1068 (size=12301) 2024-11-22T23:14:27,731 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/edc9229dae8f49738b8b8748442f8371 2024-11-22T23:14:27,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/aa47a58bd9084501bf004764bdd561b6 is 50, key is test_row_0/C:col10/1732317266485/Put/seqid=0 2024-11-22T23:14:27,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741893_1069 (size=12301) 2024-11-22T23:14:27,752 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/aa47a58bd9084501bf004764bdd561b6 2024-11-22T23:14:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6731b7bb10a84e70b30eb2dedcfc0bbc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc 2024-11-22T23:14:27,772 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc, entries=150, sequenceid=305, filesize=12.0 K 2024-11-22T23:14:27,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/edc9229dae8f49738b8b8748442f8371 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371 2024-11-22T23:14:27,778 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371, entries=150, sequenceid=305, filesize=12.0 K 2024-11-22T23:14:27,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/aa47a58bd9084501bf004764bdd561b6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6 2024-11-22T23:14:27,785 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6, entries=150, sequenceid=305, filesize=12.0 K 2024-11-22T23:14:27,787 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 152ms, sequenceid=305, compaction requested=false 2024-11-22T23:14:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:27,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-22T23:14:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-22T23:14:27,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T23:14:27,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-11-22T23:14:27,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0880 sec 2024-11-22T23:14:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:27,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:27,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:27,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69f2a44aab12472296a5c3aa93bdd6e6 is 50, key is test_row_0/A:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:27,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741894_1070 (size=14741) 2024-11-22T23:14:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-22T23:14:27,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:27,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-22T23:14:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:27,851 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:27,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:27,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317327898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:28,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:28,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317328007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317328217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:28,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69f2a44aab12472296a5c3aa93bdd6e6 2024-11-22T23:14:28,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/07752fe3cc294e0eb20aaa43b8eddeb5 is 50, key is test_row_0/B:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:28,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741895_1071 (size=12301) 2024-11-22T23:14:28,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:28,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:28,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:28,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317328528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:28,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317328547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:28,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317328547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:28,552 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:28,553 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:28,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:28,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/07752fe3cc294e0eb20aaa43b8eddeb5 2024-11-22T23:14:28,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eecacfdb08434727b4153a07d3a07ef6 is 50, key is test_row_0/C:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:28,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741896_1072 (size=12301) 2024-11-22T23:14:28,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eecacfdb08434727b4153a07d3a07ef6 2024-11-22T23:14:28,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69f2a44aab12472296a5c3aa93bdd6e6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6 2024-11-22T23:14:28,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6, entries=200, sequenceid=320, filesize=14.4 K 2024-11-22T23:14:28,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/07752fe3cc294e0eb20aaa43b8eddeb5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5 2024-11-22T23:14:28,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T23:14:28,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/eecacfdb08434727b4153a07d3a07ef6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6 2024-11-22T23:14:28,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6, entries=150, sequenceid=320, filesize=12.0 K 2024-11-22T23:14:28,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 885ms, sequenceid=320, compaction requested=true 2024-11-22T23:14:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:28,694 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:28,694 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:28,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:28,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:28,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:28,696 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:28,696 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:28,696 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:28,696 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:28,696 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,696 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,696 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e5451439d1244d879352df06880a4171, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.6 K 2024-11-22T23:14:28,696 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/2368e7ae652a4a24afa56147bd3abcf1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=39.0 K 2024-11-22T23:14:28,697 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e5451439d1244d879352df06880a4171, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:28,697 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2368e7ae652a4a24afa56147bd3abcf1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:28,697 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting edc9229dae8f49738b8b8748442f8371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317266450 2024-11-22T23:14:28,698 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6731b7bb10a84e70b30eb2dedcfc0bbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317266450 2024-11-22T23:14:28,698 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 07752fe3cc294e0eb20aaa43b8eddeb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267689 2024-11-22T23:14:28,698 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69f2a44aab12472296a5c3aa93bdd6e6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267681 2024-11-22T23:14:28,706 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:28,706 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/aba199fb31c6464d95832df1a8051919 is 50, key is test_row_0/B:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:28,708 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:28,708 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6e200c4fb0fe44ef8d46a3b08ab8a930 is 50, key is test_row_0/A:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741897_1073 (size=12983) 2024-11-22T23:14:28,724 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/aba199fb31c6464d95832df1a8051919 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/aba199fb31c6464d95832df1a8051919 2024-11-22T23:14:28,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741898_1074 (size=12983) 2024-11-22T23:14:28,734 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into aba199fb31c6464d95832df1a8051919(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:28,735 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:28,735 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317268694; duration=0sec 2024-11-22T23:14:28,736 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:28,736 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:28,737 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:28,738 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:28,738 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:28,738 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,739 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/62b21b9f5e974173a343f47814dc196f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.6 K 2024-11-22T23:14:28,739 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b21b9f5e974173a343f47814dc196f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732317265972 2024-11-22T23:14:28,739 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting aa47a58bd9084501bf004764bdd561b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317266450 2024-11-22T23:14:28,740 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting eecacfdb08434727b4153a07d3a07ef6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267689 2024-11-22T23:14:28,743 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6e200c4fb0fe44ef8d46a3b08ab8a930 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e200c4fb0fe44ef8d46a3b08ab8a930 2024-11-22T23:14:28,753 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 6e200c4fb0fe44ef8d46a3b08ab8a930(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:28,753 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:28,753 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317268694; duration=0sec 2024-11-22T23:14:28,754 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:28,754 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:28,758 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:28,760 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b09744a650ed4848ae2d4c7b8d418724 is 50, key is test_row_0/C:col10/1732317267808/Put/seqid=0 2024-11-22T23:14:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741899_1075 (size=12983) 2024-11-22T23:14:28,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-22T23:14:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:28,774 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:28,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/303ab8e14d8741e9a960218280b30409 is 50, key is test_row_0/A:col10/1732317267863/Put/seqid=0 2024-11-22T23:14:28,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741900_1076 (size=12301) 2024-11-22T23:14:28,788 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/303ab8e14d8741e9a960218280b30409 2024-11-22T23:14:28,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4302be5f393e4a47855e70daf6ecfb45 is 50, key is test_row_0/B:col10/1732317267863/Put/seqid=0 2024-11-22T23:14:28,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741901_1077 (size=12301) 2024-11-22T23:14:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:29,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:29,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317329068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:29,174 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b09744a650ed4848ae2d4c7b8d418724 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b09744a650ed4848ae2d4c7b8d418724 2024-11-22T23:14:29,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317329178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:29,181 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into b09744a650ed4848ae2d4c7b8d418724(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:29,181 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,181 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317268695; duration=0sec 2024-11-22T23:14:29,181 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:29,181 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:29,201 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4302be5f393e4a47855e70daf6ecfb45 2024-11-22T23:14:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/2eb065d6666d45ad8c2c70c1005139f8 is 50, key is test_row_0/C:col10/1732317267863/Put/seqid=0 2024-11-22T23:14:29,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741902_1078 (size=12301) 2024-11-22T23:14:29,224 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/2eb065d6666d45ad8c2c70c1005139f8 2024-11-22T23:14:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/303ab8e14d8741e9a960218280b30409 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409 2024-11-22T23:14:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:14:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4302be5f393e4a47855e70daf6ecfb45 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45 2024-11-22T23:14:29,253 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:14:29,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/2eb065d6666d45ad8c2c70c1005139f8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8 2024-11-22T23:14:29,260 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:14:29,261 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 487ms, sequenceid=342, compaction requested=false 2024-11-22T23:14:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-22T23:14:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-22T23:14:29,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T23:14:29,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-22T23:14:29,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4170 sec 2024-11-22T23:14:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:29,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:14:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:29,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:29,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 is 50, key is test_row_0/A:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741903_1079 (size=12301) 2024-11-22T23:14:29,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 2024-11-22T23:14:29,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/a2e3eb5fe0124f5ca812286a667e076a is 50, key is test_row_0/B:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741904_1080 (size=12301) 2024-11-22T23:14:29,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/a2e3eb5fe0124f5ca812286a667e076a 2024-11-22T23:14:29,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7f7a0fc2d686490f91d64f39784425a8 is 50, key is test_row_0/C:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741905_1081 (size=12301) 2024-11-22T23:14:29,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7f7a0fc2d686490f91d64f39784425a8 2024-11-22T23:14:29,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 2024-11-22T23:14:29,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-22T23:14:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/a2e3eb5fe0124f5ca812286a667e076a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a 2024-11-22T23:14:29,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a, entries=150, sequenceid=356, filesize=12.0 K 2024-11-22T23:14:29,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7f7a0fc2d686490f91d64f39784425a8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8 2024-11-22T23:14:29,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8, entries=150, sequenceid=356, filesize=12.0 K 2024-11-22T23:14:29,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 79ms, sequenceid=356, compaction requested=true 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:29,468 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:29,468 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:29,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:29,469 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:29,469 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:29,470 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:29,470 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:29,470 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:29,470 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e200c4fb0fe44ef8d46a3b08ab8a930, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.7 K 2024-11-22T23:14:29,472 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:29,472 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/aba199fb31c6464d95832df1a8051919, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.7 K 2024-11-22T23:14:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:29,475 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting aba199fb31c6464d95832df1a8051919, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267689 2024-11-22T23:14:29,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:14:29,475 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e200c4fb0fe44ef8d46a3b08ab8a930, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267689 2024-11-22T23:14:29,475 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4302be5f393e4a47855e70daf6ecfb45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732317267863 2024-11-22T23:14:29,476 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 303ab8e14d8741e9a960218280b30409, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732317267863 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:29,476 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e3eb5fe0124f5ca812286a667e076a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,476 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f0bd52d81424fb7bcc9cf6b7e50cdf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:29,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:29,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69138e4f049845339b55dfacf1792fe1 is 50, key is test_row_0/A:col10/1732317269460/Put/seqid=0 2024-11-22T23:14:29,498 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:29,498 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/8f93ee8533fa426abac312a25ef462d8 is 50, key is test_row_0/A:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,500 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:29,501 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4c6df541a8f14ae0a8021274e3fdf76d is 50, key is test_row_0/B:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741906_1082 (size=14741) 2024-11-22T23:14:29,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741907_1083 (size=13085) 2024-11-22T23:14:29,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741908_1084 (size=13085) 2024-11-22T23:14:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317329524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317329637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:29,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:29,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317329847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:29,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:29,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69138e4f049845339b55dfacf1792fe1 2024-11-22T23:14:29,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4d2e75fbde7442899e920fbcdb1432b5 is 50, key is test_row_0/B:col10/1732317269460/Put/seqid=0 2024-11-22T23:14:29,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741909_1085 (size=12301) 2024-11-22T23:14:29,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4d2e75fbde7442899e920fbcdb1432b5 2024-11-22T23:14:29,931 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/8f93ee8533fa426abac312a25ef462d8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8f93ee8533fa426abac312a25ef462d8 2024-11-22T23:14:29,936 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4c6df541a8f14ae0a8021274e3fdf76d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4c6df541a8f14ae0a8021274e3fdf76d 2024-11-22T23:14:29,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/33e5e1498fb348bca2e2bc5d43926e00 is 50, key is test_row_0/C:col10/1732317269460/Put/seqid=0 2024-11-22T23:14:29,940 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 8f93ee8533fa426abac312a25ef462d8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:29,940 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,940 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317269468; duration=0sec 2024-11-22T23:14:29,940 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:29,940 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:29,940 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:29,943 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:29,943 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:29,943 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:29,944 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b09744a650ed4848ae2d4c7b8d418724, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.7 K 2024-11-22T23:14:29,945 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b09744a650ed4848ae2d4c7b8d418724, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732317267689 2024-11-22T23:14:29,946 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb065d6666d45ad8c2c70c1005139f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732317267863 2024-11-22T23:14:29,947 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 4c6df541a8f14ae0a8021274e3fdf76d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:29,947 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f7a0fc2d686490f91d64f39784425a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:29,947 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,947 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317269468; duration=0sec 2024-11-22T23:14:29,947 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:29,947 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741910_1086 (size=12301) 2024-11-22T23:14:29,965 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#74 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:29,966 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/70e0195c8d284e65b409d9a8eaf92943 is 50, key is test_row_0/C:col10/1732317269066/Put/seqid=0 2024-11-22T23:14:29,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741911_1087 (size=13085) 2024-11-22T23:14:29,982 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/70e0195c8d284e65b409d9a8eaf92943 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/70e0195c8d284e65b409d9a8eaf92943 2024-11-22T23:14:29,989 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 70e0195c8d284e65b409d9a8eaf92943(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:29,989 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:29,989 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317269468; duration=0sec 2024-11-22T23:14:29,989 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:29,989 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T23:14:29,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:29,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-22T23:14:30,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:30,001 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:30,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:14:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:14:30,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:14:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317330158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:14:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:30,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:14:30,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/33e5e1498fb348bca2e2bc5d43926e00 2024-11-22T23:14:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/69138e4f049845339b55dfacf1792fe1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1 2024-11-22T23:14:30,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1, entries=200, sequenceid=376, filesize=14.4 K 2024-11-22T23:14:30,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4d2e75fbde7442899e920fbcdb1432b5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5 2024-11-22T23:14:30,388 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5, entries=150, sequenceid=376, filesize=12.0 K 2024-11-22T23:14:30,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/33e5e1498fb348bca2e2bc5d43926e00 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00 2024-11-22T23:14:30,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00, entries=150, sequenceid=376, filesize=12.0 K 2024-11-22T23:14:30,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 927ms, sequenceid=376, compaction requested=false 2024-11-22T23:14:30,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:30,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-22T23:14:30,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,462 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:30,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:30,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/09da9729700f480da1337f444f61f0b6 is 50, key is test_row_0/A:col10/1732317269518/Put/seqid=0 2024-11-22T23:14:30,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741912_1088 (size=12301) 2024-11-22T23:14:30,475 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/09da9729700f480da1337f444f61f0b6 2024-11-22T23:14:30,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8b9d480ee26b443c9a7d9e1c9ed17aef is 50, key is test_row_0/B:col10/1732317269518/Put/seqid=0 2024-11-22T23:14:30,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741913_1089 (size=12301) 2024-11-22T23:14:30,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:30,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317330619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317330620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:14:30,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317330668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317330727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317330728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317330937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317330937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:30,950 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8b9d480ee26b443c9a7d9e1c9ed17aef 2024-11-22T23:14:30,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7d8b654c607f46b1a53f6e02672555c9 is 50, key is test_row_0/C:col10/1732317269518/Put/seqid=0 2024-11-22T23:14:30,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741914_1090 (size=12301) 2024-11-22T23:14:30,972 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7d8b654c607f46b1a53f6e02672555c9 2024-11-22T23:14:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/09da9729700f480da1337f444f61f0b6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6 2024-11-22T23:14:30,983 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6, entries=150, sequenceid=391, filesize=12.0 K 2024-11-22T23:14:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8b9d480ee26b443c9a7d9e1c9ed17aef as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef 2024-11-22T23:14:30,990 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef, entries=150, sequenceid=391, filesize=12.0 K 2024-11-22T23:14:30,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7d8b654c607f46b1a53f6e02672555c9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9 2024-11-22T23:14:30,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9, entries=150, sequenceid=391, filesize=12.0 K 2024-11-22T23:14:30,998 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 536ms, sequenceid=391, compaction requested=true 2024-11-22T23:14:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-22T23:14:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-22T23:14:31,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-22T23:14:31,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-22T23:14:31,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0050 sec 2024-11-22T23:14:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:31,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T23:14:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:31,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:31,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cc956934e7414b9daff8c648478e72aa is 50, key is test_row_0/A:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:31,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741915_1091 (size=14741) 2024-11-22T23:14:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317331069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317331070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-22T23:14:31,138 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:31,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-22T23:14:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:31,140 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:31,141 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:31,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317331177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317331187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317331248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317331248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317331387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317331398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cc956934e7414b9daff8c648478e72aa 2024-11-22T23:14:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:31,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/639fca3ab35246e485c20c39343d32ce is 50, key is test_row_0/B:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:31,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741916_1092 (size=12301) 2024-11-22T23:14:31,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317331687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,688 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317331697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317331708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:31,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317331758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317331758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:31,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:31,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/639fca3ab35246e485c20c39343d32ce 2024-11-22T23:14:31,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b10aae5380d54eebb8606c8671f6abcf is 50, key is test_row_0/C:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:31,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741917_1093 (size=12301) 2024-11-22T23:14:31,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b10aae5380d54eebb8606c8671f6abcf 2024-11-22T23:14:31,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cc956934e7414b9daff8c648478e72aa as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa 2024-11-22T23:14:31,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa, entries=200, sequenceid=412, filesize=14.4 K 2024-11-22T23:14:31,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/639fca3ab35246e485c20c39343d32ce as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce 2024-11-22T23:14:31,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T23:14:31,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b10aae5380d54eebb8606c8671f6abcf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf 2024-11-22T23:14:31,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:31,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:31,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf, entries=150, sequenceid=412, filesize=12.0 K 2024-11-22T23:14:31,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 871ms, sequenceid=412, compaction requested=true 2024-11-22T23:14:31,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:31,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:31,910 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:31,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:31,910 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:31,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:31,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:31,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:31,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:31,912 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:31,912 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:31,912 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,912 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8f93ee8533fa426abac312a25ef462d8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=53.6 K 2024-11-22T23:14:31,912 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:31,912 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:31,912 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:31,912 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f93ee8533fa426abac312a25ef462d8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:31,913 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4c6df541a8f14ae0a8021274e3fdf76d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=48.8 K 2024-11-22T23:14:31,913 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69138e4f049845339b55dfacf1792fe1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732317269440 2024-11-22T23:14:31,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c6df541a8f14ae0a8021274e3fdf76d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:31,913 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09da9729700f480da1337f444f61f0b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732317269497 2024-11-22T23:14:31,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d2e75fbde7442899e920fbcdb1432b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732317269457 2024-11-22T23:14:31,913 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc956934e7414b9daff8c648478e72aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270612 2024-11-22T23:14:31,914 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b9d480ee26b443c9a7d9e1c9ed17aef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732317269497 2024-11-22T23:14:31,914 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 639fca3ab35246e485c20c39343d32ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270617 2024-11-22T23:14:31,929 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:31,929 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/deb19e4e2628462a91ddc0c6a5bf98c4 is 50, key is test_row_0/B:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:31,934 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:31,934 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/84661aa9fa0e47d882fd98546b56dc03 is 50, key is test_row_0/A:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:31,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741918_1094 (size=13221) 2024-11-22T23:14:31,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741919_1095 (size=13221) 2024-11-22T23:14:32,002 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/84661aa9fa0e47d882fd98546b56dc03 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/84661aa9fa0e47d882fd98546b56dc03 2024-11-22T23:14:32,012 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 84661aa9fa0e47d882fd98546b56dc03(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:32,012 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:32,012 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=12, startTime=1732317271910; duration=0sec 2024-11-22T23:14:32,012 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:32,012 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:32,012 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:32,014 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:32,014 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:32,014 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:32,014 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/70e0195c8d284e65b409d9a8eaf92943, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=48.8 K 2024-11-22T23:14:32,015 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e0195c8d284e65b409d9a8eaf92943, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732317269058 2024-11-22T23:14:32,015 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e5e1498fb348bca2e2bc5d43926e00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732317269457 2024-11-22T23:14:32,015 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d8b654c607f46b1a53f6e02672555c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732317269497 2024-11-22T23:14:32,016 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b10aae5380d54eebb8606c8671f6abcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270617 2024-11-22T23:14:32,027 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:32,027 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/cb27467fde6f4f4ab7a8d7c6d34897be is 50, key is test_row_0/C:col10/1732317270617/Put/seqid=0 2024-11-22T23:14:32,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741920_1096 (size=13221) 2024-11-22T23:14:32,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-22T23:14:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:32,061 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/49db7d1869414965987a161b3d5203c1 is 50, key is test_row_0/A:col10/1732317271060/Put/seqid=0 2024-11-22T23:14:32,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741921_1097 (size=12301) 2024-11-22T23:14:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:32,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:32,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317332261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317272238, max timestamp=1732317272269Num uniques 859; 2024-11-22T23:14:32,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:32,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317272238, max timestamp=1732317272269Num uniques 859; 2024-11-22T23:14:32,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317272238, max timestamp=1732317272269Num uniques 859; 2024-11-22T23:14:32,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:32,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:32,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317332348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,369 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/deb19e4e2628462a91ddc0c6a5bf98c4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/deb19e4e2628462a91ddc0c6a5bf98c4 2024-11-22T23:14:32,375 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into deb19e4e2628462a91ddc0c6a5bf98c4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:32,376 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:32,376 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=12, startTime=1732317271910; duration=0sec 2024-11-22T23:14:32,376 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:32,376 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:32,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317332377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,442 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/cb27467fde6f4f4ab7a8d7c6d34897be as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/cb27467fde6f4f4ab7a8d7c6d34897be 2024-11-22T23:14:32,451 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into cb27467fde6f4f4ab7a8d7c6d34897be(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:32,451 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:32,452 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=12, startTime=1732317271911; duration=0sec 2024-11-22T23:14:32,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:32,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317332468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,478 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/49db7d1869414965987a161b3d5203c1 2024-11-22T23:14:32,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/808a0cd7660e42c38a1cf89669633ff4 is 50, key is test_row_0/B:col10/1732317271060/Put/seqid=0 2024-11-22T23:14:32,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741922_1098 (size=12301) 2024-11-22T23:14:32,510 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/808a0cd7660e42c38a1cf89669633ff4 2024-11-22T23:14:32,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/c3562667b384464a836ccee48fb9a1f4 is 50, key is test_row_0/C:col10/1732317271060/Put/seqid=0 2024-11-22T23:14:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741923_1099 (size=12301) 2024-11-22T23:14:32,542 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/c3562667b384464a836ccee48fb9a1f4 2024-11-22T23:14:32,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/49db7d1869414965987a161b3d5203c1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1 2024-11-22T23:14:32,560 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1, entries=150, sequenceid=424, filesize=12.0 K 2024-11-22T23:14:32,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/808a0cd7660e42c38a1cf89669633ff4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4 2024-11-22T23:14:32,569 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4, entries=150, sequenceid=424, filesize=12.0 K 2024-11-22T23:14:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/c3562667b384464a836ccee48fb9a1f4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4 2024-11-22T23:14:32,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4, entries=150, sequenceid=424, filesize=12.0 K 2024-11-22T23:14:32,587 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 526ms, sequenceid=424, compaction requested=false 2024-11-22T23:14:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-22T23:14:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-22T23:14:32,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T23:14:32,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4480 sec 2024-11-22T23:14:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:32,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:32,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:32,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.4530 sec 2024-11-22T23:14:32,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/aafb6fc646734a888f20ed17146f217d is 50, key is test_row_0/A:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:32,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741924_1100 (size=14741) 2024-11-22T23:14:32,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/aafb6fc646734a888f20ed17146f217d 2024-11-22T23:14:32,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/82b74490cbc444ae814757d17de2ea46 is 50, key is test_row_0/B:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:32,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317332642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741925_1101 (size=12301) 2024-11-22T23:14:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317332678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317332758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317332767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,771 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317332777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,782 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317332977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:32,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317332987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:32,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/82b74490cbc444ae814757d17de2ea46 2024-11-22T23:14:33,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d551b10b27814ad092d0013f811cf39d is 50, key is test_row_0/C:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:33,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741926_1102 (size=12301) 2024-11-22T23:14:33,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d551b10b27814ad092d0013f811cf39d 2024-11-22T23:14:33,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/aafb6fc646734a888f20ed17146f217d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d 2024-11-22T23:14:33,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d, entries=200, sequenceid=461, filesize=14.4 K 2024-11-22T23:14:33,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/82b74490cbc444ae814757d17de2ea46 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46 2024-11-22T23:14:33,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46, entries=150, sequenceid=461, filesize=12.0 K 2024-11-22T23:14:33,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d551b10b27814ad092d0013f811cf39d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d 2024-11-22T23:14:33,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d, entries=150, sequenceid=461, filesize=12.0 K 2024-11-22T23:14:33,100 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 507ms, sequenceid=461, compaction requested=true 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:33,101 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:33,101 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:33,102 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:33,102 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:33,102 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:33,102 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:33,102 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,102 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,102 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/84661aa9fa0e47d882fd98546b56dc03, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=39.3 K 2024-11-22T23:14:33,102 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/deb19e4e2628462a91ddc0c6a5bf98c4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.9 K 2024-11-22T23:14:33,103 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84661aa9fa0e47d882fd98546b56dc03, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270617 2024-11-22T23:14:33,103 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting deb19e4e2628462a91ddc0c6a5bf98c4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270617 2024-11-22T23:14:33,103 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49db7d1869414965987a161b3d5203c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732317271052 2024-11-22T23:14:33,103 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 808a0cd7660e42c38a1cf89669633ff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732317271052 2024-11-22T23:14:33,104 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting aafb6fc646734a888f20ed17146f217d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272312 2024-11-22T23:14:33,104 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 82b74490cbc444ae814757d17de2ea46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272339 2024-11-22T23:14:33,114 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:33,115 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/aab8dd5e1325463292c22c86bf60e372 is 50, key is test_row_0/A:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:33,116 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:33,117 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/cb6a3564a1374a70b82bf23c347923dc is 50, key is test_row_0/B:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:33,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741927_1103 (size=13323) 2024-11-22T23:14:33,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741928_1104 (size=13323) 2024-11-22T23:14:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-22T23:14:33,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:33,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-22T23:14:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:14:33,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:33,291 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:33,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:33,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:33,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:14:33,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:33,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:33,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5f9b9d4c591e48e9a4ca9e1e8e57580d is 50, key is test_row_0/A:col10/1732317273290/Put/seqid=0 2024-11-22T23:14:33,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741929_1105 (size=14741) 2024-11-22T23:14:33,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317273343, max timestamp=1732317273360Num uniques 859; 2024-11-22T23:14:33,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:33,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317273343, max timestamp=1732317273360Num uniques 859; 2024-11-22T23:14:33,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:33,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317273343, max timestamp=1732317273360Num uniques 859; 2024-11-22T23:14:33,363 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:14:33,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317333449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317333498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,531 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/aab8dd5e1325463292c22c86bf60e372 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aab8dd5e1325463292c22c86bf60e372 2024-11-22T23:14:33,533 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/cb6a3564a1374a70b82bf23c347923dc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/cb6a3564a1374a70b82bf23c347923dc 2024-11-22T23:14:33,544 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into aab8dd5e1325463292c22c86bf60e372(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:33,545 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:33,545 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=13, startTime=1732317273101; duration=0sec 2024-11-22T23:14:33,545 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:33,545 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:33,545 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:33,546 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into cb6a3564a1374a70b82bf23c347923dc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:33,546 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:33,546 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=13, startTime=1732317273101; duration=0sec 2024-11-22T23:14:33,546 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:33,546 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:33,546 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:33,546 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:33,546 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,547 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/cb27467fde6f4f4ab7a8d7c6d34897be, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=36.9 K 2024-11-22T23:14:33,547 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb27467fde6f4f4ab7a8d7c6d34897be, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732317270617 2024-11-22T23:14:33,547 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3562667b384464a836ccee48fb9a1f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732317271052 2024-11-22T23:14:33,548 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d551b10b27814ad092d0013f811cf39d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272339 2024-11-22T23:14:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317333558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,566 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:33,566 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/20234581e20548c687766032e98e3c7d is 50, key is test_row_0/C:col10/1732317272591/Put/seqid=0 2024-11-22T23:14:33,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741930_1106 (size=13323) 2024-11-22T23:14:33,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:14:33,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5f9b9d4c591e48e9a4ca9e1e8e57580d 2024-11-22T23:14:33,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317333707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:33,711 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/178e172f1667493dab3c000b6987ba3a is 50, key is test_row_0/B:col10/1732317273290/Put/seqid=0 2024-11-22T23:14:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741931_1107 (size=12301) 2024-11-22T23:14:33,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/178e172f1667493dab3c000b6987ba3a 2024-11-22T23:14:33,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/64f1b8bf877e4d4197b122796062740e is 50, key is test_row_0/C:col10/1732317273290/Put/seqid=0 2024-11-22T23:14:33,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741932_1108 (size=12301) 2024-11-22T23:14:33,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317333767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:33,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:33,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:14:33,975 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/20234581e20548c687766032e98e3c7d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/20234581e20548c687766032e98e3c7d 2024-11-22T23:14:33,981 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 20234581e20548c687766032e98e3c7d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:33,981 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:33,981 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=13, startTime=1732317273101; duration=0sec 2024-11-22T23:14:33,982 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:33,982 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:34,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317334078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/64f1b8bf877e4d4197b122796062740e 2024-11-22T23:14:34,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/5f9b9d4c591e48e9a4ca9e1e8e57580d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d 2024-11-22T23:14:34,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d, entries=200, sequenceid=472, filesize=14.4 K 2024-11-22T23:14:34,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/178e172f1667493dab3c000b6987ba3a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a 2024-11-22T23:14:34,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a, entries=150, sequenceid=472, filesize=12.0 K 2024-11-22T23:14:34,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/64f1b8bf877e4d4197b122796062740e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e 2024-11-22T23:14:34,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e, entries=150, sequenceid=472, filesize=12.0 K 2024-11-22T23:14:34,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 874ms, sequenceid=472, compaction requested=false 2024-11-22T23:14:34,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-22T23:14:34,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-22T23:14:34,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:34,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b70899177a6c40d5922733e7a5fe1ecd is 50, key is test_row_0/A:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741933_1109 (size=12301) 2024-11-22T23:14:34,222 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b70899177a6c40d5922733e7a5fe1ecd 2024-11-22T23:14:34,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/90e46206bd5a494db4260437f50f8006 is 50, key is test_row_0/B:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741934_1110 (size=12301) 2024-11-22T23:14:34,236 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/90e46206bd5a494db4260437f50f8006 2024-11-22T23:14:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/3947e17a62ec4215947229b9a4e7f904 is 50, key is test_row_0/C:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741935_1111 (size=12301) 2024-11-22T23:14:34,259 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/3947e17a62ec4215947229b9a4e7f904 2024-11-22T23:14:34,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b70899177a6c40d5922733e7a5fe1ecd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd 2024-11-22T23:14:34,279 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd, entries=150, sequenceid=511, filesize=12.0 K 2024-11-22T23:14:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/90e46206bd5a494db4260437f50f8006 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006 2024-11-22T23:14:34,286 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006, entries=150, sequenceid=511, filesize=12.0 K 2024-11-22T23:14:34,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/3947e17a62ec4215947229b9a4e7f904 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904 2024-11-22T23:14:34,296 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904, entries=150, sequenceid=511, filesize=12.0 K 2024-11-22T23:14:34,297 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 85ms, sequenceid=511, compaction requested=true 2024-11-22T23:14:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-22T23:14:34,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-22T23:14:34,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T23:14:34,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-22T23:14:34,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.0180 sec 2024-11-22T23:14:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T23:14:34,427 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:34,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:34,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-22T23:14:34,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:34,431 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:34,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:34,432 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:34,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:34,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:34,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:34,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/39b8baba54484701ae511592ba0dc0bb is 50, key is test_row_1/A:col10/1732317274524/Put/seqid=0 2024-11-22T23:14:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:34,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741936_1112 (size=14737) 2024-11-22T23:14:34,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/39b8baba54484701ae511592ba0dc0bb 2024-11-22T23:14:34,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/01a6a789e7cf47b997615434a3194e68 is 50, key is test_row_1/B:col10/1732317274524/Put/seqid=0 2024-11-22T23:14:34,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741937_1113 (size=9857) 2024-11-22T23:14:34,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/01a6a789e7cf47b997615434a3194e68 2024-11-22T23:14:34,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/13b6e2e90d1b4ee8880b7673fc166e1c is 50, key is test_row_1/C:col10/1732317274524/Put/seqid=0 2024-11-22T23:14:34,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741938_1114 (size=9857) 2024-11-22T23:14:34,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317274544, max timestamp=1732317274617Num uniques 859; 2024-11-22T23:14:34,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317274544, max timestamp=1732317274617Num uniques 859; 2024-11-22T23:14:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317274544, max timestamp=1732317274617Num uniques 859; 2024-11-22T23:14:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:14:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:14:34,621 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:14:34,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/13b6e2e90d1b4ee8880b7673fc166e1c 2024-11-22T23:14:34,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/39b8baba54484701ae511592ba0dc0bb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb 2024-11-22T23:14:34,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb, entries=200, sequenceid=521, filesize=14.4 K 2024-11-22T23:14:34,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/01a6a789e7cf47b997615434a3194e68 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68 2024-11-22T23:14:34,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68, entries=100, sequenceid=521, filesize=9.6 K 2024-11-22T23:14:34,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/13b6e2e90d1b4ee8880b7673fc166e1c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c 2024-11-22T23:14:34,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c, entries=100, sequenceid=521, filesize=9.6 K 2024-11-22T23:14:34,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 144ms, sequenceid=521, compaction requested=true 2024-11-22T23:14:34,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,671 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:34,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:34,672 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:34,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:34,673 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55102 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:34,673 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:34,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=114.05 KB heapSize=315.84 KB 2024-11-22T23:14:34,674 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,674 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aab8dd5e1325463292c22c86bf60e372, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=53.8 K 2024-11-22T23:14:34,674 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:34,674 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:34,674 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,674 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting aab8dd5e1325463292c22c86bf60e372, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272339 2024-11-22T23:14:34,674 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/cb6a3564a1374a70b82bf23c347923dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=46.7 K 2024-11-22T23:14:34,675 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f9b9d4c591e48e9a4ca9e1e8e57580d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732317272612 2024-11-22T23:14:34,675 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting cb6a3564a1374a70b82bf23c347923dc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272339 2024-11-22T23:14:34,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:34,675 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70899177a6c40d5922733e7a5fe1ecd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732317273423 2024-11-22T23:14:34,675 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 178e172f1667493dab3c000b6987ba3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732317272620 2024-11-22T23:14:34,676 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b8baba54484701ae511592ba0dc0bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317274516 2024-11-22T23:14:34,676 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e46206bd5a494db4260437f50f8006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732317273423 2024-11-22T23:14:34,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a6a789e7cf47b997615434a3194e68, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317274516 2024-11-22T23:14:34,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:34,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:34,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:34,696 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:34,697 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b0eea9f725484e088c1a10bad78847f5 is 50, key is test_row_0/A:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,707 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:34,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a254cef96f384437b587035377fd866f is 50, key is test_row_0/A:col10/1732317274657/Put/seqid=0 2024-11-22T23:14:34,708 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/c379e7679c5e43cab52d23864ae7cbcf is 50, key is test_row_0/B:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317334721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317334722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741939_1115 (size=13459) 2024-11-22T23:14:34,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,744 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b0eea9f725484e088c1a10bad78847f5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b0eea9f725484e088c1a10bad78847f5 2024-11-22T23:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:34,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741940_1116 (size=17181) 2024-11-22T23:14:34,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a254cef96f384437b587035377fd866f 2024-11-22T23:14:34,755 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into b0eea9f725484e088c1a10bad78847f5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:34,755 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,755 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=12, startTime=1732317274671; duration=0sec 2024-11-22T23:14:34,756 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:34,756 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:34,756 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:34,759 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:34,759 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:34,759 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,759 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/20234581e20548c687766032e98e3c7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=46.7 K 2024-11-22T23:14:34,763 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20234581e20548c687766032e98e3c7d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1732317272339 2024-11-22T23:14:34,764 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f1b8bf877e4d4197b122796062740e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1732317272620 2024-11-22T23:14:34,764 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3947e17a62ec4215947229b9a4e7f904, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1732317273423 2024-11-22T23:14:34,765 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b6e2e90d1b4ee8880b7673fc166e1c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317274516 2024-11-22T23:14:34,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741941_1117 (size=13459) 2024-11-22T23:14:34,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b5ddd575cc954c348e603c47eb0d7a89 is 50, key is test_row_0/B:col10/1732317274657/Put/seqid=0 2024-11-22T23:14:34,781 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/c379e7679c5e43cab52d23864ae7cbcf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/c379e7679c5e43cab52d23864ae7cbcf 2024-11-22T23:14:34,792 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into c379e7679c5e43cab52d23864ae7cbcf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:34,792 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,792 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=12, startTime=1732317274672; duration=0sec 2024-11-22T23:14:34,793 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:34,793 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:34,793 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:34,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46484 deadline: 1732317334788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46528 deadline: 1732317334788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,794 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,794 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 4a71851c051c02efdfd8dcb5ec6b9ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,796 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/a7557dc286fd452fa310b0c60d59160f is 50, key is test_row_0/C:col10/1732317273423/Put/seqid=0 2024-11-22T23:14:34,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741942_1118 (size=12301) 2024-11-22T23:14:34,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b5ddd575cc954c348e603c47eb0d7a89 2024-11-22T23:14:34,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741943_1119 (size=13459) 2024-11-22T23:14:34,837 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/a7557dc286fd452fa310b0c60d59160f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a7557dc286fd452fa310b0c60d59160f 2024-11-22T23:14:34,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/74ba4da5e3314e28adc686832fdb0553 is 50, key is test_row_0/C:col10/1732317274657/Put/seqid=0 2024-11-22T23:14:34,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317334837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317334839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:34,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:34,845 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into a7557dc286fd452fa310b0c60d59160f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:34,845 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:34,845 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=12, startTime=1732317274672; duration=0sec 2024-11-22T23:14:34,846 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:34,846 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:34,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741944_1120 (size=12301) 2024-11-22T23:14:34,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:34,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:34,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:34,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317335047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317335047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:35,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:35,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:35,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:35,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/74ba4da5e3314e28adc686832fdb0553 2024-11-22T23:14:35,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/a254cef96f384437b587035377fd866f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f 2024-11-22T23:14:35,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f, entries=250, sequenceid=553, filesize=16.8 K 2024-11-22T23:14:35,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/b5ddd575cc954c348e603c47eb0d7a89 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89 2024-11-22T23:14:35,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89, entries=150, sequenceid=553, filesize=12.0 K 2024-11-22T23:14:35,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/74ba4da5e3314e28adc686832fdb0553 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553 2024-11-22T23:14:35,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553, entries=150, sequenceid=553, filesize=12.0 K 2024-11-22T23:14:35,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~315.80 KB/323376, currentSize=67.09 KB/68700 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 617ms, sequenceid=553, compaction requested=false 2024-11-22T23:14:35,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:35,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-22T23:14:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,356 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:35,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:35,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6e7f63b2d2f84a868abe1bac0b6ad5ef is 50, key is test_row_0/A:col10/1732317274718/Put/seqid=0 2024-11-22T23:14:35,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741945_1121 (size=12301) 2024-11-22T23:14:35,395 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6e7f63b2d2f84a868abe1bac0b6ad5ef 2024-11-22T23:14:35,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317335413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317335417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8c36121ed8b04328b71739c3c24a4df0 is 50, key is test_row_0/B:col10/1732317274718/Put/seqid=0 2024-11-22T23:14:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741946_1122 (size=12301) 2024-11-22T23:14:35,449 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8c36121ed8b04328b71739c3c24a4df0 2024-11-22T23:14:35,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/de0e7105fd1b4c5db8276ff19843c2bb is 50, key is test_row_0/C:col10/1732317274718/Put/seqid=0 2024-11-22T23:14:35,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741947_1123 (size=12301) 2024-11-22T23:14:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317335527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317335527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317335738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317335739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:35,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:35,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/de0e7105fd1b4c5db8276ff19843c2bb 2024-11-22T23:14:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/6e7f63b2d2f84a868abe1bac0b6ad5ef as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef 2024-11-22T23:14:35,948 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef, entries=150, sequenceid=569, filesize=12.0 K 2024-11-22T23:14:35,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/8c36121ed8b04328b71739c3c24a4df0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0 2024-11-22T23:14:35,956 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0, entries=150, sequenceid=569, filesize=12.0 K 2024-11-22T23:14:35,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/de0e7105fd1b4c5db8276ff19843c2bb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb 2024-11-22T23:14:35,966 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb, entries=150, sequenceid=569, filesize=12.0 K 2024-11-22T23:14:35,967 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 611ms, sequenceid=569, compaction requested=true 2024-11-22T23:14:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-22T23:14:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-22T23:14:35,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-22T23:14:35,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5380 sec 2024-11-22T23:14:35,977 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5470 sec 2024-11-22T23:14:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:36,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T23:14:36,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317275395, max timestamp=1732317276049Num uniques 859; 2024-11-22T23:14:36,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:36,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317275395, max timestamp=1732317276049Num uniques 859; 2024-11-22T23:14:36,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317275395, max timestamp=1732317276049Num uniques 859; 2024-11-22T23:14:36,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:36,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:36,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:36,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b46cbccc3353447ab46d0e341ff6751f is 50, key is test_row_0/A:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741948_1124 (size=17181) 2024-11-22T23:14:36,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b46cbccc3353447ab46d0e341ff6751f 2024-11-22T23:14:36,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d26d3de0b00948f5a1b7e233df8ed6ac is 50, key is test_row_0/B:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741949_1125 (size=12301) 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317276099, max timestamp=1732317276110Num uniques 859; 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317276099, max timestamp=1732317276110Num uniques 859; 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317276099, max timestamp=1732317276110Num uniques 859; 2024-11-22T23:14:36,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732317276155, max timestamp=1732317276176Num uniques 1068; 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732317276155, max timestamp=1732317276176Num uniques 1068; 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732317276155, max timestamp=1732317276176Num uniques 1068; 2024-11-22T23:14:36,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732317276225, max timestamp=1732317276255Num uniques 1077; 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732317276225, max timestamp=1732317276255Num uniques 1077; 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732317276225, max timestamp=1732317276255Num uniques 1077; 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-22T23:14:36,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-22T23:14:36,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317336331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317336332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317336447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317336447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d26d3de0b00948f5a1b7e233df8ed6ac 2024-11-22T23:14:36,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7a71295646884b4aa205d2fe0cd44e64 is 50, key is test_row_0/C:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741950_1126 (size=12301) 2024-11-22T23:14:36,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7a71295646884b4aa205d2fe0cd44e64 2024-11-22T23:14:36,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/b46cbccc3353447ab46d0e341ff6751f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f 2024-11-22T23:14:36,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f, entries=250, sequenceid=591, filesize=16.8 K 2024-11-22T23:14:36,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/d26d3de0b00948f5a1b7e233df8ed6ac as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac 2024-11-22T23:14:36,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac, entries=150, sequenceid=591, filesize=12.0 K 2024-11-22T23:14:36,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/7a71295646884b4aa205d2fe0cd44e64 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64 2024-11-22T23:14:36,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64, entries=150, sequenceid=591, filesize=12.0 K 2024-11-22T23:14:36,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 497ms, sequenceid=591, compaction requested=true 2024-11-22T23:14:36,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:36,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:36,549 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:36,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:36,549 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:36,553 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 60122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:36,553 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/A is initiating minor compaction (all files) 2024-11-22T23:14:36,553 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/A in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,553 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b0eea9f725484e088c1a10bad78847f5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=58.7 K 2024-11-22T23:14:36,554 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:36,554 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/B is initiating minor compaction (all files) 2024-11-22T23:14:36,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/B in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/c379e7679c5e43cab52d23864ae7cbcf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=49.2 K 2024-11-22T23:14:36,555 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0eea9f725484e088c1a10bad78847f5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317273423 2024-11-22T23:14:36,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:36,556 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c379e7679c5e43cab52d23864ae7cbcf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317273423 2024-11-22T23:14:36,556 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ddd575cc954c348e603c47eb0d7a89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1732317274654 2024-11-22T23:14:36,559 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a254cef96f384437b587035377fd866f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1732317274643 2024-11-22T23:14:36,560 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c36121ed8b04328b71739c3c24a4df0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732317274711 2024-11-22T23:14:36,560 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7f63b2d2f84a868abe1bac0b6ad5ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732317274711 2024-11-22T23:14:36,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:36,560 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d26d3de0b00948f5a1b7e233df8ed6ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732317275395 2024-11-22T23:14:36,561 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b46cbccc3353447ab46d0e341ff6751f, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732317275395 2024-11-22T23:14:36,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:36,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:36,587 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#A#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:36,588 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1ab86989532d489e81b948ed8e96279b is 50, key is test_row_0/A:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-22T23:14:36,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:36,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-22T23:14:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:36,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:36,593 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:36,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:36,603 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:36,604 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4b845cf355e24b23a7c4786c3af1798d is 50, key is test_row_0/B:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741952_1128 (size=13595) 2024-11-22T23:14:36,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741951_1127 (size=13595) 2024-11-22T23:14:36,627 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/4b845cf355e24b23a7c4786c3af1798d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4b845cf355e24b23a7c4786c3af1798d 2024-11-22T23:14:36,635 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/1ab86989532d489e81b948ed8e96279b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1ab86989532d489e81b948ed8e96279b 2024-11-22T23:14:36,637 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/B of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 4b845cf355e24b23a7c4786c3af1798d(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:36,637 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:36,638 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/B, priority=12, startTime=1732317276549; duration=0sec 2024-11-22T23:14:36,638 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:36,638 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:36,638 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:36,641 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:36,642 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 4a71851c051c02efdfd8dcb5ec6b9ff7/C is initiating minor compaction (all files) 2024-11-22T23:14:36,642 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4a71851c051c02efdfd8dcb5ec6b9ff7/C in TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,642 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a7557dc286fd452fa310b0c60d59160f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp, totalSize=49.2 K 2024-11-22T23:14:36,642 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a7557dc286fd452fa310b0c60d59160f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732317273423 2024-11-22T23:14:36,643 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ba4da5e3314e28adc686832fdb0553, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1732317274654 2024-11-22T23:14:36,643 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting de0e7105fd1b4c5db8276ff19843c2bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1732317274711 2024-11-22T23:14:36,644 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a71295646884b4aa205d2fe0cd44e64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732317275395 2024-11-22T23:14:36,647 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/A of 4a71851c051c02efdfd8dcb5ec6b9ff7 into 1ab86989532d489e81b948ed8e96279b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:36,647 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:36,647 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/A, priority=12, startTime=1732317276549; duration=0sec 2024-11-22T23:14:36,649 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:36,649 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:36,659 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4a71851c051c02efdfd8dcb5ec6b9ff7#C#compaction#116 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:36,659 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d41bf9fbec084e419a7ea0083c529186 is 50, key is test_row_0/C:col10/1732317276049/Put/seqid=0 2024-11-22T23:14:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:36,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=161.02 KB heapSize=449.63 KB 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:36,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:14:36,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/df52204eaa4542129a364f033f13e257 is 50, key is test_row_0/A:col10/1732317276322/Put/seqid=0 2024-11-22T23:14:36,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741953_1129 (size=13595) 2024-11-22T23:14:36,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317336675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317336680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,696 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d41bf9fbec084e419a7ea0083c529186 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d41bf9fbec084e419a7ea0083c529186 2024-11-22T23:14:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:36,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741954_1130 (size=12301) 2024-11-22T23:14:36,707 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4a71851c051c02efdfd8dcb5ec6b9ff7/C of 4a71851c051c02efdfd8dcb5ec6b9ff7 into d41bf9fbec084e419a7ea0083c529186(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:36,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:36,707 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., storeName=4a71851c051c02efdfd8dcb5ec6b9ff7/C, priority=12, startTime=1732317276560; duration=0sec 2024-11-22T23:14:36,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:36,707 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:36,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317336787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317336788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:36,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:36,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:36,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:36,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:36,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317336999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317336999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/df52204eaa4542129a364f033f13e257 2024-11-22T23:14:37,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/1029615a03d94a8390709bcadf5ea1d7 is 50, key is test_row_0/B:col10/1732317276322/Put/seqid=0 2024-11-22T23:14:37,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741955_1131 (size=12301) 2024-11-22T23:14:37,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:37,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:37,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:37,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317337307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317337307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:37,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:37,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:37,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/1029615a03d94a8390709bcadf5ea1d7 2024-11-22T23:14:37,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d5abbbb0dc9749e7afa7ded30d3903bf is 50, key is test_row_0/C:col10/1732317276322/Put/seqid=0 2024-11-22T23:14:37,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741956_1132 (size=12301) 2024-11-22T23:14:37,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=665 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d5abbbb0dc9749e7afa7ded30d3903bf 2024-11-22T23:14:37,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/df52204eaa4542129a364f033f13e257 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/df52204eaa4542129a364f033f13e257 2024-11-22T23:14:37,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/df52204eaa4542129a364f033f13e257, entries=150, sequenceid=665, filesize=12.0 K 2024-11-22T23:14:37,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/1029615a03d94a8390709bcadf5ea1d7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/1029615a03d94a8390709bcadf5ea1d7 2024-11-22T23:14:37,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/1029615a03d94a8390709bcadf5ea1d7, entries=150, sequenceid=665, filesize=12.0 K 2024-11-22T23:14:37,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/d5abbbb0dc9749e7afa7ded30d3903bf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d5abbbb0dc9749e7afa7ded30d3903bf 2024-11-22T23:14:37,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d5abbbb0dc9749e7afa7ded30d3903bf, entries=150, sequenceid=665, filesize=12.0 K 2024-11-22T23:14:37,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~449.58 KB/460368, currentSize=26.84 KB/27480 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 901ms, sequenceid=665, compaction requested=false 2024-11-22T23:14:37,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:37,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-22T23:14:37,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:37,669 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:37,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:37,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cee5dfaaaf7b438da0803de7ed9b89e6 is 50, key is test_row_0/A:col10/1732317276674/Put/seqid=0 2024-11-22T23:14:37,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741957_1133 (size=9857) 2024-11-22T23:14:37,691 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cee5dfaaaf7b438da0803de7ed9b89e6 2024-11-22T23:14:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/0343729fb05a44b18c51f0f1f6108ea2 is 50, key is test_row_0/B:col10/1732317276674/Put/seqid=0 2024-11-22T23:14:37,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741958_1134 (size=9857) 2024-11-22T23:14:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:37,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. as already flushing 2024-11-22T23:14:37,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317277767, max timestamp=1732317277790Num uniques 859; 2024-11-22T23:14:37,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:37,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317337820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317337821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317337822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317337938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317337938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317337938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:37,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,118 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/0343729fb05a44b18c51f0f1f6108ea2 2024-11-22T23:14:38,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/32593a0b6dc0475b9ea151904660c868 is 50, key is test_row_0/C:col10/1732317276674/Put/seqid=0 2024-11-22T23:14:38,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741959_1135 (size=9857) 2024-11-22T23:14:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317338147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317338148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317338148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,459 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-22T23:14:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46490 deadline: 1732317338457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,459 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,459 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-22T23:14:38,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,459 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46512 deadline: 1732317338457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:38,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46488 deadline: 1732317338457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:14:38,465 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,465 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,465 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,465 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,466 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,468 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-22T23:14:38,469 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,469 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,469 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,475 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-22T23:14:38,475 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,475 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,475 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,532 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/32593a0b6dc0475b9ea151904660c868 2024-11-22T23:14:38,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/cee5dfaaaf7b438da0803de7ed9b89e6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cee5dfaaaf7b438da0803de7ed9b89e6 2024-11-22T23:14:38,547 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cee5dfaaaf7b438da0803de7ed9b89e6, entries=100, sequenceid=673, filesize=9.6 K 2024-11-22T23:14:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/0343729fb05a44b18c51f0f1f6108ea2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/0343729fb05a44b18c51f0f1f6108ea2 2024-11-22T23:14:38,553 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/0343729fb05a44b18c51f0f1f6108ea2, entries=100, sequenceid=673, filesize=9.6 K 2024-11-22T23:14:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/32593a0b6dc0475b9ea151904660c868 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/32593a0b6dc0475b9ea151904660c868 2024-11-22T23:14:38,558 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/32593a0b6dc0475b9ea151904660c868, entries=100, sequenceid=673, filesize=9.6 K 2024-11-22T23:14:38,559 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=152.07 KB/155720 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 890ms, sequenceid=673, compaction requested=true 2024-11-22T23:14:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-22T23:14:38,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-22T23:14:38,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T23:14:38,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-11-22T23:14:38,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 1.9740 sec 2024-11-22T23:14:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-22T23:14:38,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:38,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=165.49 KB heapSize=486.33 KB 2024-11-22T23:14:38,819 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-22T23:14:38,819 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-22T23:14:38,819 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,819 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,819 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:38,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:38,819 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,819 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:38,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:38,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:38,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:38,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9d1316281e9d47f29f1443e8d3ae758c is 50, key is test_row_0/A:col10/1732317277821/Put/seqid=0 2024-11-22T23:14:38,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741960_1136 (size=12301) 2024-11-22T23:14:38,973 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-22T23:14:38,973 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,973 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,973 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-22T23:14:38,973 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,974 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,974 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,974 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,974 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,975 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-22T23:14:38,975 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:14:38,976 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,976 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-22T23:14:38,976 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2549 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2552 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3383 rows 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1135 2024-11-22T23:14:38,977 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3399 rows 2024-11-22T23:14:38,977 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:14:38,977 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:14:38,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:38,977 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:14:38,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:14:38,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:38,988 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317278988"}]},"ts":"1732317278988"} 2024-11-22T23:14:38,990 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:14:38,990 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:14:38,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:14:38,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, UNASSIGN}] 2024-11-22T23:14:38,997 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, UNASSIGN 2024-11-22T23:14:38,998 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=4a71851c051c02efdfd8dcb5ec6b9ff7, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:39,000 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, UNASSIGN because future has completed 2024-11-22T23:14:39,001 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:14:39,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:39,161 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:39,162 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:14:39,164 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 4a71851c051c02efdfd8dcb5ec6b9ff7, disabling compactions & flushes 2024-11-22T23:14:39,164 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:39,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9d1316281e9d47f29f1443e8d3ae758c 2024-11-22T23:14:39,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/56f20b1131f3442db73c7db0de6469d5 is 50, key is test_row_0/B:col10/1732317277821/Put/seqid=0 2024-11-22T23:14:39,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741961_1137 (size=12301) 2024-11-22T23:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:39,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/56f20b1131f3442db73c7db0de6469d5 2024-11-22T23:14:39,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/faf376aa3da640c7a7de48faff20fb10 is 50, key is test_row_0/C:col10/1732317277821/Put/seqid=0 2024-11-22T23:14:39,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741962_1138 (size=12301) 2024-11-22T23:14:39,807 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:14:40,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=64.85 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/faf376aa3da640c7a7de48faff20fb10 2024-11-22T23:14:40,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/9d1316281e9d47f29f1443e8d3ae758c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9d1316281e9d47f29f1443e8d3ae758c 2024-11-22T23:14:40,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9d1316281e9d47f29f1443e8d3ae758c, entries=150, sequenceid=705, filesize=12.0 K 2024-11-22T23:14:40,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/56f20b1131f3442db73c7db0de6469d5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/56f20b1131f3442db73c7db0de6469d5 2024-11-22T23:14:40,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/56f20b1131f3442db73c7db0de6469d5, entries=150, sequenceid=705, filesize=12.0 K 2024-11-22T23:14:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/faf376aa3da640c7a7de48faff20fb10 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/faf376aa3da640c7a7de48faff20fb10 2024-11-22T23:14:40,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/faf376aa3da640c7a7de48faff20fb10, entries=150, sequenceid=705, filesize=12.0 K 2024-11-22T23:14:40,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~165.49 KB/169460, heapSize ~486.28 KB/497952, currentSize=20.13 KB/20610 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 1288ms, sequenceid=705, compaction requested=true 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: 2024-11-22T23:14:40,108 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. after waiting 0 ms 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4a71851c051c02efdfd8dcb5ec6b9ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. because compaction request was cancelled 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. because compaction request was cancelled 2024-11-22T23:14:40,108 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-22T23:14:40,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:B 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:A 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. because compaction request was cancelled 2024-11-22T23:14:40,108 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4a71851c051c02efdfd8dcb5ec6b9ff7:C 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=A 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=B 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:40,108 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4a71851c051c02efdfd8dcb5ec6b9ff7, store=C 2024-11-22T23:14:40,109 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:40,113 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/af77833443004ffd9bc6fb397f609c55 is 50, key is test_row_0/A:col10/1732317278969/Put/seqid=0 2024-11-22T23:14:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741963_1139 (size=12301) 2024-11-22T23:14:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:40,523 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/af77833443004ffd9bc6fb397f609c55 2024-11-22T23:14:40,536 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/326a8c01230d49c2be2cbdc058edaf29 is 50, key is test_row_0/B:col10/1732317278969/Put/seqid=0 2024-11-22T23:14:40,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741964_1140 (size=12301) 2024-11-22T23:14:40,941 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/326a8c01230d49c2be2cbdc058edaf29 2024-11-22T23:14:41,008 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b0beb12f10f74df0bc2af868e0cbb1fe is 50, key is test_row_0/C:col10/1732317278969/Put/seqid=0 2024-11-22T23:14:41,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741965_1141 (size=12301) 2024-11-22T23:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:41,415 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=711 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b0beb12f10f74df0bc2af868e0cbb1fe 2024-11-22T23:14:41,425 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/A/af77833443004ffd9bc6fb397f609c55 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/af77833443004ffd9bc6fb397f609c55 2024-11-22T23:14:41,432 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/af77833443004ffd9bc6fb397f609c55, entries=150, sequenceid=711, filesize=12.0 K 2024-11-22T23:14:41,434 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/B/326a8c01230d49c2be2cbdc058edaf29 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/326a8c01230d49c2be2cbdc058edaf29 2024-11-22T23:14:41,439 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/326a8c01230d49c2be2cbdc058edaf29, entries=150, sequenceid=711, filesize=12.0 K 2024-11-22T23:14:41,440 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/.tmp/C/b0beb12f10f74df0bc2af868e0cbb1fe as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b0beb12f10f74df0bc2af868e0cbb1fe 2024-11-22T23:14:41,445 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b0beb12f10f74df0bc2af868e0cbb1fe, entries=150, sequenceid=711, filesize=12.0 K 2024-11-22T23:14:41,445 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 1337ms, sequenceid=711, compaction requested=true 2024-11-22T23:14:41,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f55234e617c408f908f0baa630afa45, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/32399f2beb1f4289b0e6f5e6fc613c74, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b914de47580b4b89829d39eaca8e1775, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fa97689d71e34068b771b2e6ce1fc1be, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/2368e7ae652a4a24afa56147bd3abcf1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e200c4fb0fe44ef8d46a3b08ab8a930, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8f93ee8533fa426abac312a25ef462d8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/84661aa9fa0e47d882fd98546b56dc03, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aab8dd5e1325463292c22c86bf60e372, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b0eea9f725484e088c1a10bad78847f5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f] to archive 2024-11-22T23:14:41,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:14:41,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/c250c15f9a1c44a0b400ac2ae4992284 2024-11-22T23:14:41,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/ecf911562e3b4b4dadad8b15723e0905 2024-11-22T23:14:41,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fe64bc678e524dd39b50492fd229e786 2024-11-22T23:14:41,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f55234e617c408f908f0baa630afa45 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f55234e617c408f908f0baa630afa45 2024-11-22T23:14:41,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/0904ec0a45954a22bc486bd57773ef68 2024-11-22T23:14:41,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/d3f8771ca59442ea8d6525943d5d9e68 2024-11-22T23:14:41,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/32399f2beb1f4289b0e6f5e6fc613c74 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/32399f2beb1f4289b0e6f5e6fc613c74 2024-11-22T23:14:41,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a3ff9a3dcf2e432eb66c4a7a1bace89c 2024-11-22T23:14:41,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8c7272603c1e4449bee697d47233cbbc 2024-11-22T23:14:41,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b914de47580b4b89829d39eaca8e1775 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b914de47580b4b89829d39eaca8e1775 2024-11-22T23:14:41,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/90f2cd6ddf694a5e8d952b9c0cfd8181 2024-11-22T23:14:41,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/3260f3f5a2564392b1349e64677a1e19 2024-11-22T23:14:41,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fa97689d71e34068b771b2e6ce1fc1be to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/fa97689d71e34068b771b2e6ce1fc1be 2024-11-22T23:14:41,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9c8290de913947a988baa48efa63734c 2024-11-22T23:14:41,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/06c6baa5db024f7dbe8508e5ee23686b 2024-11-22T23:14:41,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b2e98bedd78146e8ba27219bbe43b076 2024-11-22T23:14:41,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5c96f1a3a0f84c579871c754f6969dee 2024-11-22T23:14:41,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/2368e7ae652a4a24afa56147bd3abcf1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/2368e7ae652a4a24afa56147bd3abcf1 2024-11-22T23:14:41,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6731b7bb10a84e70b30eb2dedcfc0bbc 2024-11-22T23:14:41,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69f2a44aab12472296a5c3aa93bdd6e6 2024-11-22T23:14:41,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e200c4fb0fe44ef8d46a3b08ab8a930 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e200c4fb0fe44ef8d46a3b08ab8a930 2024-11-22T23:14:41,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/303ab8e14d8741e9a960218280b30409 2024-11-22T23:14:41,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8f93ee8533fa426abac312a25ef462d8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/8f93ee8533fa426abac312a25ef462d8 2024-11-22T23:14:41,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1f0bd52d81424fb7bcc9cf6b7e50cdf6 2024-11-22T23:14:41,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/69138e4f049845339b55dfacf1792fe1 2024-11-22T23:14:41,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/09da9729700f480da1337f444f61f0b6 2024-11-22T23:14:41,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cc956934e7414b9daff8c648478e72aa 2024-11-22T23:14:41,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/84661aa9fa0e47d882fd98546b56dc03 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/84661aa9fa0e47d882fd98546b56dc03 2024-11-22T23:14:41,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/49db7d1869414965987a161b3d5203c1 2024-11-22T23:14:41,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aafb6fc646734a888f20ed17146f217d 2024-11-22T23:14:41,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aab8dd5e1325463292c22c86bf60e372 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/aab8dd5e1325463292c22c86bf60e372 2024-11-22T23:14:41,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/5f9b9d4c591e48e9a4ca9e1e8e57580d 2024-11-22T23:14:41,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b70899177a6c40d5922733e7a5fe1ecd 2024-11-22T23:14:41,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/39b8baba54484701ae511592ba0dc0bb 2024-11-22T23:14:41,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b0eea9f725484e088c1a10bad78847f5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b0eea9f725484e088c1a10bad78847f5 2024-11-22T23:14:41,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/a254cef96f384437b587035377fd866f 2024-11-22T23:14:41,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/6e7f63b2d2f84a868abe1bac0b6ad5ef 2024-11-22T23:14:41,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/b46cbccc3353447ab46d0e341ff6751f 2024-11-22T23:14:41,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/480a90870213465b93a28481ff1a5705, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/6c6abb015aa4449a94d1435d6582b44e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8d04d0666d9b44dd90f0ec0af600d553, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/58450bb1987642ca9b8d12c9428c68a2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e5451439d1244d879352df06880a4171, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/aba199fb31c6464d95832df1a8051919, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4c6df541a8f14ae0a8021274e3fdf76d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/deb19e4e2628462a91ddc0c6a5bf98c4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/cb6a3564a1374a70b82bf23c347923dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/c379e7679c5e43cab52d23864ae7cbcf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac] to archive 2024-11-22T23:14:41,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:14:41,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/588f935c82d3463bb5a9c5e2adacc5e4 2024-11-22T23:14:41,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/9a09347220bb4f22956b7a6246e302e5 2024-11-22T23:14:41,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/480a90870213465b93a28481ff1a5705 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/480a90870213465b93a28481ff1a5705 2024-11-22T23:14:41,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e02e2f124b1c46dfa9353dbe96358ebb 2024-11-22T23:14:41,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5b13e782be946deb5600f3b15b844a0 2024-11-22T23:14:41,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/6c6abb015aa4449a94d1435d6582b44e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/6c6abb015aa4449a94d1435d6582b44e 2024-11-22T23:14:41,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b1d00a0cf9fb4a9ea0a155b20bd801eb 2024-11-22T23:14:41,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d5d84840976e4226b6bbbaefb3da40fb 2024-11-22T23:14:41,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8d04d0666d9b44dd90f0ec0af600d553 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8d04d0666d9b44dd90f0ec0af600d553 2024-11-22T23:14:41,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/f61a051d75bd4afe84ac66bbfcb98bae 2024-11-22T23:14:41,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/497c212f0ce6497fa922f57465c40ea9 2024-11-22T23:14:41,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/58450bb1987642ca9b8d12c9428c68a2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/58450bb1987642ca9b8d12c9428c68a2 2024-11-22T23:14:41,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b6b27aac7a7a4cbdb7c92c83c64b5a87 2024-11-22T23:14:41,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/831316dac444466db7bb687ea71a4a36 2024-11-22T23:14:41,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/ac35c3b373e244d2b644ebe70ef9c6b7 2024-11-22T23:14:41,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d14d292b189745798ecdce3c7f47b581 2024-11-22T23:14:41,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e5451439d1244d879352df06880a4171 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/e5451439d1244d879352df06880a4171 2024-11-22T23:14:41,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d2aa13b6990e451ebd0429a1ba38917a 2024-11-22T23:14:41,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/edc9229dae8f49738b8b8748442f8371 2024-11-22T23:14:41,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/aba199fb31c6464d95832df1a8051919 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/aba199fb31c6464d95832df1a8051919 2024-11-22T23:14:41,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/07752fe3cc294e0eb20aaa43b8eddeb5 2024-11-22T23:14:41,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4302be5f393e4a47855e70daf6ecfb45 2024-11-22T23:14:41,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4c6df541a8f14ae0a8021274e3fdf76d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4c6df541a8f14ae0a8021274e3fdf76d 2024-11-22T23:14:41,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/a2e3eb5fe0124f5ca812286a667e076a 2024-11-22T23:14:41,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4d2e75fbde7442899e920fbcdb1432b5 2024-11-22T23:14:41,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8b9d480ee26b443c9a7d9e1c9ed17aef 2024-11-22T23:14:41,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/deb19e4e2628462a91ddc0c6a5bf98c4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/deb19e4e2628462a91ddc0c6a5bf98c4 2024-11-22T23:14:41,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/639fca3ab35246e485c20c39343d32ce 2024-11-22T23:14:41,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/808a0cd7660e42c38a1cf89669633ff4 2024-11-22T23:14:41,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/cb6a3564a1374a70b82bf23c347923dc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/cb6a3564a1374a70b82bf23c347923dc 2024-11-22T23:14:41,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/82b74490cbc444ae814757d17de2ea46 2024-11-22T23:14:41,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/178e172f1667493dab3c000b6987ba3a 2024-11-22T23:14:41,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/90e46206bd5a494db4260437f50f8006 2024-11-22T23:14:41,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/c379e7679c5e43cab52d23864ae7cbcf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/c379e7679c5e43cab52d23864ae7cbcf 2024-11-22T23:14:41,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/01a6a789e7cf47b997615434a3194e68 2024-11-22T23:14:41,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/b5ddd575cc954c348e603c47eb0d7a89 2024-11-22T23:14:41,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/8c36121ed8b04328b71739c3c24a4df0 2024-11-22T23:14:41,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/d26d3de0b00948f5a1b7e233df8ed6ac 2024-11-22T23:14:41,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b28091fff33b4e42a7e69746c0e5501c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/932c00f6b740419aa9deb3179038048b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/fb4d57bdf3824ce5b8dc9bcd183e3774, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/83be63cb77804ad7b73c1634d3be52f1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/62b21b9f5e974173a343f47814dc196f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b09744a650ed4848ae2d4c7b8d418724, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/70e0195c8d284e65b409d9a8eaf92943, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/cb27467fde6f4f4ab7a8d7c6d34897be, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/20234581e20548c687766032e98e3c7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a7557dc286fd452fa310b0c60d59160f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64] to archive 2024-11-22T23:14:41,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:14:41,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/f5a2307a616846fe80681a5da8b81d70 2024-11-22T23:14:41,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5fd01d2b59fa4f2f88a6ee38e383c452 2024-11-22T23:14:41,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b28091fff33b4e42a7e69746c0e5501c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b28091fff33b4e42a7e69746c0e5501c 2024-11-22T23:14:41,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eba02b0876cb4f2fb8a55f0284506b5c 2024-11-22T23:14:41,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e79024e739f5472f8105bb9b35ae9709 2024-11-22T23:14:41,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/932c00f6b740419aa9deb3179038048b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/932c00f6b740419aa9deb3179038048b 2024-11-22T23:14:41,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/87ea5b57c31f475e8897687fe4ca6d9b 2024-11-22T23:14:41,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/22f9655d2996432db7a7f8d364192f4e 2024-11-22T23:14:41,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/fb4d57bdf3824ce5b8dc9bcd183e3774 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/fb4d57bdf3824ce5b8dc9bcd183e3774 2024-11-22T23:14:41,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e3f3f1ca7e10480191b405f27fa2648e 2024-11-22T23:14:41,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/69f7db8059534927a91e45e47babf5b3 2024-11-22T23:14:41,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/83be63cb77804ad7b73c1634d3be52f1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/83be63cb77804ad7b73c1634d3be52f1 2024-11-22T23:14:41,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/e8aabbd1ad674b9aa82c7782da19275c 2024-11-22T23:14:41,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a69c573532e940209ff271bfc3728585 2024-11-22T23:14:41,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f0397bafd214145ab59d0fa3afee5fe 2024-11-22T23:14:41,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/5f49a7191d1046d4ae98469888b362a6 2024-11-22T23:14:41,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/62b21b9f5e974173a343f47814dc196f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/62b21b9f5e974173a343f47814dc196f 2024-11-22T23:14:41,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/ea0b03eaa3254a07a7cce5424d73bfe7 2024-11-22T23:14:41,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/aa47a58bd9084501bf004764bdd561b6 2024-11-22T23:14:41,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b09744a650ed4848ae2d4c7b8d418724 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b09744a650ed4848ae2d4c7b8d418724 2024-11-22T23:14:41,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/eecacfdb08434727b4153a07d3a07ef6 2024-11-22T23:14:41,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/2eb065d6666d45ad8c2c70c1005139f8 2024-11-22T23:14:41,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/70e0195c8d284e65b409d9a8eaf92943 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/70e0195c8d284e65b409d9a8eaf92943 2024-11-22T23:14:41,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7f7a0fc2d686490f91d64f39784425a8 2024-11-22T23:14:41,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/33e5e1498fb348bca2e2bc5d43926e00 2024-11-22T23:14:41,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7d8b654c607f46b1a53f6e02672555c9 2024-11-22T23:14:41,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/cb27467fde6f4f4ab7a8d7c6d34897be to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/cb27467fde6f4f4ab7a8d7c6d34897be 2024-11-22T23:14:41,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b10aae5380d54eebb8606c8671f6abcf 2024-11-22T23:14:41,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/c3562667b384464a836ccee48fb9a1f4 2024-11-22T23:14:41,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/20234581e20548c687766032e98e3c7d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/20234581e20548c687766032e98e3c7d 2024-11-22T23:14:41,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d551b10b27814ad092d0013f811cf39d 2024-11-22T23:14:41,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/64f1b8bf877e4d4197b122796062740e 2024-11-22T23:14:41,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/3947e17a62ec4215947229b9a4e7f904 2024-11-22T23:14:41,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a7557dc286fd452fa310b0c60d59160f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/a7557dc286fd452fa310b0c60d59160f 2024-11-22T23:14:41,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/13b6e2e90d1b4ee8880b7673fc166e1c 2024-11-22T23:14:41,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/74ba4da5e3314e28adc686832fdb0553 2024-11-22T23:14:41,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/de0e7105fd1b4c5db8276ff19843c2bb 2024-11-22T23:14:41,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/7a71295646884b4aa205d2fe0cd44e64 2024-11-22T23:14:41,587 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/recovered.edits/714.seqid, newMaxSeqId=714, maxSeqId=1 2024-11-22T23:14:41,588 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. 2024-11-22T23:14:41,588 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 4a71851c051c02efdfd8dcb5ec6b9ff7: Waiting for close lock at 1732317279163Running coprocessor pre-close hooks at 1732317279163Disabling compacts and flushes for region at 1732317279164 (+1 ms)Disabling writes for close at 1732317280108 (+944 ms)Obtaining lock to block concurrent updates at 1732317280108Preparing flush snapshotting stores in 4a71851c051c02efdfd8dcb5ec6b9ff7 at 1732317280108Finished memstore snapshotting TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732317280109 (+1 ms)Flushing stores of TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7. at 1732317280109Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/A: creating writer at 1732317280109Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/A: appending metadata at 1732317280112 (+3 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/A: closing flushed file at 1732317280113 (+1 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/B: creating writer at 1732317280532 (+419 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/B: appending metadata at 1732317280535 (+3 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/B: closing flushed file at 1732317280535Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/C: creating writer at 1732317281003 (+468 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/C: appending metadata at 1732317281007 (+4 ms)Flushing 4a71851c051c02efdfd8dcb5ec6b9ff7/C: closing flushed file at 1732317281007Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fd6519f: reopening flushed file at 1732317281424 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78fe56e0: reopening flushed file at 1732317281432 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@672bb1e1: reopening flushed file at 1732317281439 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4a71851c051c02efdfd8dcb5ec6b9ff7 in 1337ms, sequenceid=711, compaction requested=true at 1732317281445 (+6 ms)Writing region close event to WAL at 1732317281583 (+138 ms)Running coprocessor post-close hooks at 1732317281587 (+4 ms)Closed at 1732317281587 2024-11-22T23:14:41,590 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:41,591 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=4a71851c051c02efdfd8dcb5ec6b9ff7, regionState=CLOSED 2024-11-22T23:14:41,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:14:41,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-22T23:14:41,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 4a71851c051c02efdfd8dcb5ec6b9ff7, server=a3d87a7361f3,46715,1732317251834 in 2.5920 sec 2024-11-22T23:14:41,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-22T23:14:41,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4a71851c051c02efdfd8dcb5ec6b9ff7, UNASSIGN in 2.5990 sec 2024-11-22T23:14:41,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T23:14:41,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6050 sec 2024-11-22T23:14:41,601 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317281600"}]},"ts":"1732317281600"} 2024-11-22T23:14:41,602 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:14:41,602 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:14:41,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6210 sec 2024-11-22T23:14:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T23:14:43,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:43,151 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:14:43,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-22T23:14:43,158 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,163 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:43,167 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/recovered.edits] 2024-11-22T23:14:43,170 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1ab86989532d489e81b948ed8e96279b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/1ab86989532d489e81b948ed8e96279b 2024-11-22T23:14:43,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9d1316281e9d47f29f1443e8d3ae758c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/9d1316281e9d47f29f1443e8d3ae758c 2024-11-22T23:14:43,173 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/af77833443004ffd9bc6fb397f609c55 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/af77833443004ffd9bc6fb397f609c55 2024-11-22T23:14:43,174 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cee5dfaaaf7b438da0803de7ed9b89e6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/cee5dfaaaf7b438da0803de7ed9b89e6 2024-11-22T23:14:43,175 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/df52204eaa4542129a364f033f13e257 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/A/df52204eaa4542129a364f033f13e257 2024-11-22T23:14:43,178 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/0343729fb05a44b18c51f0f1f6108ea2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/0343729fb05a44b18c51f0f1f6108ea2 2024-11-22T23:14:43,179 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/1029615a03d94a8390709bcadf5ea1d7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/1029615a03d94a8390709bcadf5ea1d7 2024-11-22T23:14:43,180 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/326a8c01230d49c2be2cbdc058edaf29 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/326a8c01230d49c2be2cbdc058edaf29 2024-11-22T23:14:43,181 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4b845cf355e24b23a7c4786c3af1798d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/4b845cf355e24b23a7c4786c3af1798d 2024-11-22T23:14:43,182 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/56f20b1131f3442db73c7db0de6469d5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/B/56f20b1131f3442db73c7db0de6469d5 2024-11-22T23:14:43,185 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/32593a0b6dc0475b9ea151904660c868 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/32593a0b6dc0475b9ea151904660c868 2024-11-22T23:14:43,186 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b0beb12f10f74df0bc2af868e0cbb1fe to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/b0beb12f10f74df0bc2af868e0cbb1fe 2024-11-22T23:14:43,187 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d41bf9fbec084e419a7ea0083c529186 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d41bf9fbec084e419a7ea0083c529186 2024-11-22T23:14:43,189 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d5abbbb0dc9749e7afa7ded30d3903bf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/d5abbbb0dc9749e7afa7ded30d3903bf 2024-11-22T23:14:43,190 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/faf376aa3da640c7a7de48faff20fb10 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/C/faf376aa3da640c7a7de48faff20fb10 2024-11-22T23:14:43,192 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/recovered.edits/714.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7/recovered.edits/714.seqid 2024-11-22T23:14:43,193 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/4a71851c051c02efdfd8dcb5ec6b9ff7 2024-11-22T23:14:43,193 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:14:43,198 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-22T23:14:43,207 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:14:43,237 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:14:43,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:14:43,239 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317283239"}]},"ts":"9223372036854775807"} 2024-11-22T23:14:43,243 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:14:43,243 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4a71851c051c02efdfd8dcb5ec6b9ff7, NAME => 'TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:14:43,244 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:14:43,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317283244"}]},"ts":"9223372036854775807"} 2024-11-22T23:14:43,247 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:14:43,248 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-22T23:14:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-22T23:14:43,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:14:43,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:43,281 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_889432899_22 at /127.0.0.1:44392 [Waiting for operation #290] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;a3d87a7361f3:46715-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1769707779_22 at /127.0.0.1:44570 [Waiting for operation #277] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_889432899_22 at /127.0.0.1:44666 [Waiting for operation #240] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=388 (was 288) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=857 (was 1426) 2024-11-22T23:14:43,290 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=388, ProcessCount=11, AvailableMemoryMB=856 2024-11-22T23:14:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:14:43,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:14:43,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:43,294 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:14:43,294 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:43,294 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-22T23:14:43,295 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:14:43,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T23:14:43,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741966_1142 (size=1061) 2024-11-22T23:14:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T23:14:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T23:14:43,703 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:14:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741967_1143 (size=53) 2024-11-22T23:14:43,714 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:43,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f65aabacfa7d7a5aa320670d2d93e1d0, disabling compactions & flushes 2024-11-22T23:14:43,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:43,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:43,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. after waiting 0 ms 2024-11-22T23:14:43,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:43,715 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:43,715 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f65aabacfa7d7a5aa320670d2d93e1d0: Waiting for close lock at 1732317283714Disabling compacts and flushes for region at 1732317283714Disabling writes for close at 1732317283715 (+1 ms)Writing region close event to WAL at 1732317283715Closed at 1732317283715 2024-11-22T23:14:43,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:14:43,717 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317283717"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317283717"}]},"ts":"1732317283717"} 2024-11-22T23:14:43,720 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:14:43,721 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:14:43,722 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317283721"}]},"ts":"1732317283721"} 2024-11-22T23:14:43,724 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:14:43,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, ASSIGN}] 2024-11-22T23:14:43,728 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, ASSIGN 2024-11-22T23:14:43,729 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:14:43,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:43,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, ASSIGN because future has completed 2024-11-22T23:14:43,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T23:14:44,039 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:44,039 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:14:44,040 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,040 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:44,040 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,040 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,042 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,043 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:44,044 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName A 2024-11-22T23:14:44,044 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:44,044 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:44,045 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,046 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:44,046 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName B 2024-11-22T23:14:44,046 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:44,046 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:44,046 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,047 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:44,047 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName C 2024-11-22T23:14:44,047 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:44,048 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:44,048 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:44,048 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,049 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,049 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,050 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,050 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,051 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:14:44,052 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,054 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:14:44,054 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened f65aabacfa7d7a5aa320670d2d93e1d0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71479921, jitterRate=0.06513382494449615}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:14:44,054 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:44,055 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for f65aabacfa7d7a5aa320670d2d93e1d0: Running coprocessor pre-open hook at 1732317284040Writing region info on filesystem at 1732317284040Initializing all the Stores at 1732317284041 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317284041Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317284041Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317284041Cleaning up temporary data from old regions at 1732317284050 (+9 ms)Running coprocessor post-open hooks at 1732317284054 (+4 ms)Region opened successfully at 1732317284055 (+1 ms) 2024-11-22T23:14:44,056 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., pid=38, masterSystemTime=1732317284035 2024-11-22T23:14:44,058 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:44,059 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:44,059 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:44,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:14:44,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-22T23:14:44,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 in 179 msec 2024-11-22T23:14:44,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-22T23:14:44,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, ASSIGN in 339 msec 2024-11-22T23:14:44,068 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:14:44,068 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317284068"}]},"ts":"1732317284068"} 2024-11-22T23:14:44,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:14:44,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:14:44,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 780 msec 2024-11-22T23:14:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-22T23:14:44,438 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:44,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:44,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:44,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:44,441 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:44,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af5b370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:44,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:44,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:44,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cedc1ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:44,445 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:44,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:44,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:14:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:14:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17eb0dc9 2024-11-22T23:14:44,453 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:14:44,454 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:14:44,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:14:44,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:14:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:14:44,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741968_1144 (size=1097) 2024-11-22T23:14:44,881 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:14:44,882 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:14:44,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:14:44,894 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE}] 2024-11-22T23:14:44,895 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE 2024-11-22T23:14:44,896 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:44,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE because future has completed 2024-11-22T23:14:44,898 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:14:44,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:45,052 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,052 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:14:45,052 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing f65aabacfa7d7a5aa320670d2d93e1d0, disabling compactions & flushes 2024-11-22T23:14:45,052 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,052 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,052 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. after waiting 0 ms 2024-11-22T23:14:45,053 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,061 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:14:45,062 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,062 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for f65aabacfa7d7a5aa320670d2d93e1d0: Waiting for close lock at 1732317285052Running coprocessor pre-close hooks at 1732317285052Disabling compacts and flushes for region at 1732317285052Disabling writes for close at 1732317285053 (+1 ms)Writing region close event to WAL at 1732317285054 (+1 ms)Running coprocessor post-close hooks at 1732317285062 (+8 ms)Closed at 1732317285062 2024-11-22T23:14:45,062 WARN [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: f65aabacfa7d7a5aa320670d2d93e1d0 to self. 2024-11-22T23:14:45,065 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=CLOSED 2024-11-22T23:14:45,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:14:45,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-22T23:14:45,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 in 170 msec 2024-11-22T23:14:45,071 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE; state=CLOSED, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=true 2024-11-22T23:14:45,222 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE because future has completed 2024-11-22T23:14:45,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:14:45,390 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,391 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:14:45,392 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,392 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:14:45,392 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,392 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,399 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,400 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:45,406 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName A 2024-11-22T23:14:45,409 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:45,409 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:45,410 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,411 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:45,411 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName B 2024-11-22T23:14:45,411 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:45,412 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:45,412 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,412 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:14:45,413 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65aabacfa7d7a5aa320670d2d93e1d0 columnFamilyName C 2024-11-22T23:14:45,413 DEBUG [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:45,413 INFO [StoreOpener-f65aabacfa7d7a5aa320670d2d93e1d0-1 {}] regionserver.HStore(327): Store=f65aabacfa7d7a5aa320670d2d93e1d0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:14:45,413 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,413 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,414 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,415 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,416 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,416 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,417 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:14:45,418 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,419 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f65aabacfa7d7a5aa320670d2d93e1d0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74246397, jitterRate=0.10635752975940704}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:14:45,419 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,419 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f65aabacfa7d7a5aa320670d2d93e1d0: Running coprocessor pre-open hook at 1732317285393Writing region info on filesystem at 1732317285393Initializing all the Stores at 1732317285395 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317285395Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317285395Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317285395Cleaning up temporary data from old regions at 1732317285416 (+21 ms)Running coprocessor post-open hooks at 1732317285419 (+3 ms)Region opened successfully at 1732317285419 2024-11-22T23:14:45,420 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., pid=43, masterSystemTime=1732317285383 2024-11-22T23:14:45,422 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,422 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,422 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=OPEN, openSeqNum=5, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:14:45,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-22T23:14:45,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 in 196 msec 2024-11-22T23:14:45,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-22T23:14:45,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, REOPEN/MOVE in 532 msec 2024-11-22T23:14:45,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-22T23:14:45,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 541 msec 2024-11-22T23:14:45,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 970 msec 2024-11-22T23:14:45,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-22T23:14:45,436 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:45,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,438 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d857ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,439 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d89034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,442 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5efd2437, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,442 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,443 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,443 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742ebc14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,446 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7339318d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77af0491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,450 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698e3a33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8ec05a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,454 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a6ebf9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,455 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31cf46ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,456 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,456 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,457 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1909a20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36a924bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,461 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761a8232, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,462 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,463 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7203a827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,465 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bfb8bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,466 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7825eb66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:14:45,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:14:45,469 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:14:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:14:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:14:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e804398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:14:45,469 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:14:45,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:14:45,470 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:14:45,471 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411e8aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:14:45,472 DEBUG [Thread-722 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,472 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,472 DEBUG [Thread-713 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,473 DEBUG [Thread-716 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,474 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:45,475 DEBUG [Thread-718 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,476 DEBUG [Thread-716 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,476 DEBUG [Thread-718 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,476 DEBUG [Thread-716 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-22T23:14:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:45,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:45,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:45,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:45,488 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,488 DEBUG [Thread-705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,489 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,489 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:14:45,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:14:45,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:14:45,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:14:45,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:45,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:45,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a6a55cbf7d44993831b1e22b96ee08e_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317285491/Put/seqid=0 2024-11-22T23:14:45,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741969_1145 (size=14594) 2024-11-22T23:14:45,581 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:45,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a6a55cbf7d44993831b1e22b96ee08e_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a6a55cbf7d44993831b1e22b96ee08e_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:45,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d137919e9cbc4320bf99cae4d972126c, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:45,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317285543, max timestamp=1732317285590Num uniques 859; 2024-11-22T23:14:45,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:14:45,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d137919e9cbc4320bf99cae4d972126c is 175, key is test_row_0/A:col10/1732317285491/Put/seqid=0 2024-11-22T23:14:45,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317285543, max timestamp=1732317285598Num uniques 859; 2024-11-22T23:14:45,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:45,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317285543, max timestamp=1732317285598Num uniques 859; 2024-11-22T23:14:45,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:14:45,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741970_1146 (size=39549) 2024-11-22T23:14:45,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:14:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317345640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317345645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317345645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317345648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317345648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317345758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317345758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317345758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317345759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317345768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:14:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:45,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:45,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:14:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:45,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:45,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317345978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317345979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317345979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317345979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:45,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317345988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:45,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,016 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d137919e9cbc4320bf99cae4d972126c 2024-11-22T23:14:46,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/85df224d3f6b4238be269e6db0502b16 is 50, key is test_row_0/B:col10/1732317285491/Put/seqid=0 2024-11-22T23:14:46,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741971_1147 (size=12001) 2024-11-22T23:14:46,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/85df224d3f6b4238be269e6db0502b16 2024-11-22T23:14:46,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/66d2e9052862418fafa19a8a5cca5e7d is 50, key is test_row_0/C:col10/1732317285491/Put/seqid=0 2024-11-22T23:14:46,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:14:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:46,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:46,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:46,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:46,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741972_1148 (size=12001) 2024-11-22T23:14:46,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/66d2e9052862418fafa19a8a5cca5e7d 2024-11-22T23:14:46,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d137919e9cbc4320bf99cae4d972126c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c 2024-11-22T23:14:46,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c, entries=200, sequenceid=14, filesize=38.6 K 2024-11-22T23:14:46,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/85df224d3f6b4238be269e6db0502b16 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16 2024-11-22T23:14:46,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:14:46,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/66d2e9052862418fafa19a8a5cca5e7d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d 2024-11-22T23:14:46,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-22T23:14:46,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for f65aabacfa7d7a5aa320670d2d93e1d0 in 651ms, sequenceid=14, compaction requested=false 2024-11-22T23:14:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:46,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-22T23:14:46,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:46,258 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:46,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:46,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d333fbb8cb5046cc86e6b1e8ab00a908_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317285644/Put/seqid=0 2024-11-22T23:14:46,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741973_1149 (size=12154) 2024-11-22T23:14:46,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:46,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:46,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317346305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,311 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,314 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d333fbb8cb5046cc86e6b1e8ab00a908_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d333fbb8cb5046cc86e6b1e8ab00a908_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0707bcc289a54701901e942303ec250c, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:46,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0707bcc289a54701901e942303ec250c is 175, key is test_row_0/A:col10/1732317285644/Put/seqid=0 2024-11-22T23:14:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317346305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317346306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317346308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317346309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741974_1150 (size=30955) 2024-11-22T23:14:46,330 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0707bcc289a54701901e942303ec250c 2024-11-22T23:14:46,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/ff2aa86ed0b348839d6428d49b4a89db is 50, key is test_row_0/B:col10/1732317285644/Put/seqid=0 2024-11-22T23:14:46,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741975_1151 (size=12001) 2024-11-22T23:14:46,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317346417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317346427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317346428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317346428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317346428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:46,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317346628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317346638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317346638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317346638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317346639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,755 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/ff2aa86ed0b348839d6428d49b4a89db 2024-11-22T23:14:46,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7e1416036121497b855b6213ae56889d is 50, key is test_row_0/C:col10/1732317285644/Put/seqid=0 2024-11-22T23:14:46,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741976_1152 (size=12001) 2024-11-22T23:14:46,782 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7e1416036121497b855b6213ae56889d 2024-11-22T23:14:46,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0707bcc289a54701901e942303ec250c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c 2024-11-22T23:14:46,795 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c, entries=150, sequenceid=51, filesize=30.2 K 2024-11-22T23:14:46,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/ff2aa86ed0b348839d6428d49b4a89db as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db 2024-11-22T23:14:46,804 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T23:14:46,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7e1416036121497b855b6213ae56889d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d 2024-11-22T23:14:46,817 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-22T23:14:46,820 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for f65aabacfa7d7a5aa320670d2d93e1d0 in 563ms, sequenceid=51, compaction requested=false 2024-11-22T23:14:46,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:46,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:46,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-22T23:14:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-22T23:14:46,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-22T23:14:46,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3370 sec 2024-11-22T23:14:46,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3530 sec 2024-11-22T23:14:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:46,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:46,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:46,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226ac155afb53e409aba9d0268c9900462_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732317286969, max timestamp=1732317286976Num uniques 859; 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732317286969, max timestamp=1732317286976Num uniques 859; 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732317286969, max timestamp=1732317286976Num uniques 859; 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:46,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317346974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:46,980 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:46,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:46,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741977_1153 (size=14594) 2024-11-22T23:14:46,984 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:46,992 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226ac155afb53e409aba9d0268c9900462_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226ac155afb53e409aba9d0268c9900462_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:46,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/cbc377e3c8e94739b4fce8fbc974870d, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:46,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/cbc377e3c8e94739b4fce8fbc974870d is 175, key is test_row_0/A:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:47,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317347005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317347007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317347008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741978_1154 (size=39549) 2024-11-22T23:14:47,019 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/cbc377e3c8e94739b4fce8fbc974870d 2024-11-22T23:14:47,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317347014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e47d683a13f840afb9bcd005d92e4607 is 50, key is test_row_0/B:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741979_1155 (size=12001) 2024-11-22T23:14:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317347088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317347117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317347117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317347128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317347128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,140 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:14:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317347297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317347328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317347328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317347337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317347337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e47d683a13f840afb9bcd005d92e4607 2024-11-22T23:14:47,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/2166f1e76d0e4e95a5efffd94eb4540a is 50, key is test_row_0/C:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741980_1156 (size=12001) 2024-11-22T23:14:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317347607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-22T23:14:47,628 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:47,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-22T23:14:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:47,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:47,630 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:47,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:47,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317347648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317347648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317347648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317347648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:47,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:14:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:47,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:47,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/2166f1e76d0e4e95a5efffd94eb4540a 2024-11-22T23:14:47,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/cbc377e3c8e94739b4fce8fbc974870d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d 2024-11-22T23:14:47,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d, entries=200, sequenceid=62, filesize=38.6 K 2024-11-22T23:14:47,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e47d683a13f840afb9bcd005d92e4607 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607 2024-11-22T23:14:47,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607, entries=150, sequenceid=62, filesize=11.7 K 2024-11-22T23:14:47,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/2166f1e76d0e4e95a5efffd94eb4540a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a 2024-11-22T23:14:47,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a, entries=150, sequenceid=62, filesize=11.7 K 2024-11-22T23:14:47,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f65aabacfa7d7a5aa320670d2d93e1d0 in 947ms, sequenceid=62, compaction requested=true 2024-11-22T23:14:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:47,888 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:47,888 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:47,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:47,890 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:47,890 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/A is initiating minor compaction (all files) 2024-11-22T23:14:47,890 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/A in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,890 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=107.5 K 2024-11-22T23:14:47,890 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,890 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d] 2024-11-22T23:14:47,891 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:47,891 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/B is initiating minor compaction (all files) 2024-11-22T23:14:47,891 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/B in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,891 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.2 K 2024-11-22T23:14:47,892 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 85df224d3f6b4238be269e6db0502b16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732317285491 2024-11-22T23:14:47,892 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d137919e9cbc4320bf99cae4d972126c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732317285491 2024-11-22T23:14:47,893 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0707bcc289a54701901e942303ec250c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732317285641 2024-11-22T23:14:47,895 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ff2aa86ed0b348839d6428d49b4a89db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732317285641 2024-11-22T23:14:47,895 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc377e3c8e94739b4fce8fbc974870d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286303 2024-11-22T23:14:47,895 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e47d683a13f840afb9bcd005d92e4607, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286305 2024-11-22T23:14:47,903 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:47,907 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#B#compaction#139 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:47,908 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/328243e7bce84b31ad77eb722e294d65 is 50, key is test_row_0/B:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:47,917 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112206537dbe98f046ff9ce52550de8e22b0_f65aabacfa7d7a5aa320670d2d93e1d0 store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:47,922 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112206537dbe98f046ff9ce52550de8e22b0_f65aabacfa7d7a5aa320670d2d93e1d0, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:47,922 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112206537dbe98f046ff9ce52550de8e22b0_f65aabacfa7d7a5aa320670d2d93e1d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:47,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-22T23:14:47,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:47,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:47,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741981_1157 (size=12104) 2024-11-22T23:14:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:47,955 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/328243e7bce84b31ad77eb722e294d65 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/328243e7bce84b31ad77eb722e294d65 2024-11-22T23:14:47,963 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/B of f65aabacfa7d7a5aa320670d2d93e1d0 into 328243e7bce84b31ad77eb722e294d65(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:47,963 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:47,963 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/B, priority=13, startTime=1732317287888; duration=0sec 2024-11-22T23:14:47,963 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:47,963 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:B 2024-11-22T23:14:47,963 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:47,964 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:47,964 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/C is initiating minor compaction (all files) 2024-11-22T23:14:47,964 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/C in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:47,965 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.2 K 2024-11-22T23:14:47,965 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d2e9052862418fafa19a8a5cca5e7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732317285491 2024-11-22T23:14:47,965 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1416036121497b855b6213ae56889d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732317285641 2024-11-22T23:14:47,966 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2166f1e76d0e4e95a5efffd94eb4540a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286305 2024-11-22T23:14:47,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741982_1158 (size=4469) 2024-11-22T23:14:47,977 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:47,978 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/d4e9e08ad8814800bcd9818c2cfa0a0c is 50, key is test_row_0/C:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122df33558563de45c68797f0f030773a55_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317287003/Put/seqid=0 2024-11-22T23:14:47,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741983_1159 (size=12104) 2024-11-22T23:14:48,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741984_1160 (size=12154) 2024-11-22T23:14:48,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317348168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317348168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317348168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317348168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317348168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:48,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317348277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,375 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:48,376 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d25c56865fe4439e9a8a70d5cbc90fa9 is 175, key is test_row_0/A:col10/1732317286939/Put/seqid=0 2024-11-22T23:14:48,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741985_1161 (size=31058) 2024-11-22T23:14:48,388 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/d25c56865fe4439e9a8a70d5cbc90fa9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9 2024-11-22T23:14:48,394 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/A of f65aabacfa7d7a5aa320670d2d93e1d0 into d25c56865fe4439e9a8a70d5cbc90fa9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:48,394 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:48,395 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/A, priority=13, startTime=1732317287888; duration=0sec 2024-11-22T23:14:48,395 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:48,395 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:A 2024-11-22T23:14:48,400 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/d4e9e08ad8814800bcd9818c2cfa0a0c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/d4e9e08ad8814800bcd9818c2cfa0a0c 2024-11-22T23:14:48,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:48,407 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/C of f65aabacfa7d7a5aa320670d2d93e1d0 into d4e9e08ad8814800bcd9818c2cfa0a0c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:48,407 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:48,407 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/C, priority=13, startTime=1732317287889; duration=0sec 2024-11-22T23:14:48,407 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122df33558563de45c68797f0f030773a55_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122df33558563de45c68797f0f030773a55_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:48,407 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:48,407 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:C 2024-11-22T23:14:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/6cdc17d2fe7b42489ec5b5e0885682c2, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/6cdc17d2fe7b42489ec5b5e0885682c2 is 175, key is test_row_0/A:col10/1732317287003/Put/seqid=0 2024-11-22T23:14:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741986_1162 (size=30955) 2024-11-22T23:14:48,424 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/6cdc17d2fe7b42489ec5b5e0885682c2 2024-11-22T23:14:48,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fa72855165594312b9a7fc79f3f0bd53 is 50, key is test_row_0/B:col10/1732317287003/Put/seqid=0 2024-11-22T23:14:48,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741987_1163 (size=12001) 2024-11-22T23:14:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317348487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317348797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:48,852 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fa72855165594312b9a7fc79f3f0bd53 2024-11-22T23:14:48,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/68498f61b5b0419db65557b285254af6 is 50, key is test_row_0/C:col10/1732317287003/Put/seqid=0 2024-11-22T23:14:48,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741988_1164 (size=12001) 2024-11-22T23:14:49,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317349177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317349178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,183 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:49,183 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317349187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317349187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,190 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,190 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:49,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,264 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/68498f61b5b0419db65557b285254af6 2024-11-22T23:14:49,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/6cdc17d2fe7b42489ec5b5e0885682c2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2 2024-11-22T23:14:49,273 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2, entries=150, sequenceid=96, filesize=30.2 K 2024-11-22T23:14:49,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fa72855165594312b9a7fc79f3f0bd53 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53 2024-11-22T23:14:49,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53, entries=150, sequenceid=96, filesize=11.7 K 2024-11-22T23:14:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/68498f61b5b0419db65557b285254af6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6 2024-11-22T23:14:49,291 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6, entries=150, sequenceid=96, filesize=11.7 K 2024-11-22T23:14:49,292 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1357ms, sequenceid=96, compaction requested=false 2024-11-22T23:14:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-22T23:14:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-22T23:14:49,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-22T23:14:49,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-22T23:14:49,301 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6710 sec 2024-11-22T23:14:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:49,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:49,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f71be6f267e5490e9cb61c9540433b0a_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:49,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741989_1165 (size=14594) 2024-11-22T23:14:49,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317349406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317349517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317349728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:49,749 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:49,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f71be6f267e5490e9cb61c9540433b0a_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f71be6f267e5490e9cb61c9540433b0a_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:49,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/04158921d697443ea72c8a2ba2cd3cdb, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:49,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/04158921d697443ea72c8a2ba2cd3cdb is 175, key is test_row_0/A:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:49,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741990_1166 (size=39549) 2024-11-22T23:14:49,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/04158921d697443ea72c8a2ba2cd3cdb 2024-11-22T23:14:49,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e6bdeddabd1140399feab23af51e698e is 50, key is test_row_0/B:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-22T23:14:49,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741991_1167 (size=12001) 2024-11-22T23:14:49,778 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:49,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-22T23:14:49,780 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:49,781 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:49,781 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:49,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T23:14:49,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:49,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:49,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:49,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:49,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:50,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317350037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:50,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T23:14:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:50,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:50,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e6bdeddabd1140399feab23af51e698e 2024-11-22T23:14:50,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e7ab95e5c1924a7680f304f13d031ed7 is 50, key is test_row_0/C:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:50,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741992_1168 (size=12001) 2024-11-22T23:14:50,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e7ab95e5c1924a7680f304f13d031ed7 2024-11-22T23:14:50,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/04158921d697443ea72c8a2ba2cd3cdb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb 2024-11-22T23:14:50,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb, entries=200, sequenceid=111, filesize=38.6 K 2024-11-22T23:14:50,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/e6bdeddabd1140399feab23af51e698e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e 2024-11-22T23:14:50,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e, entries=150, sequenceid=111, filesize=11.7 K 2024-11-22T23:14:50,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e7ab95e5c1924a7680f304f13d031ed7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7 2024-11-22T23:14:50,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7, entries=150, sequenceid=111, filesize=11.7 K 2024-11-22T23:14:50,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f65aabacfa7d7a5aa320670d2d93e1d0 in 907ms, sequenceid=111, compaction requested=true 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:50,227 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:50,227 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:50,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:50,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:50,228 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:50,228 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:50,228 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/A is initiating minor compaction (all files) 2024-11-22T23:14:50,228 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/B is initiating minor compaction (all files) 2024-11-22T23:14:50,229 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/B in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,229 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/328243e7bce84b31ad77eb722e294d65, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.3 K 2024-11-22T23:14:50,229 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/A in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,229 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=99.2 K 2024-11-22T23:14:50,229 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,229 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb] 2024-11-22T23:14:50,229 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 328243e7bce84b31ad77eb722e294d65, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286305 2024-11-22T23:14:50,233 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25c56865fe4439e9a8a70d5cbc90fa9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286305 2024-11-22T23:14:50,234 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cdc17d2fe7b42489ec5b5e0885682c2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732317287003 2024-11-22T23:14:50,234 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fa72855165594312b9a7fc79f3f0bd53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732317287003 2024-11-22T23:14:50,234 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04158921d697443ea72c8a2ba2cd3cdb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288140 2024-11-22T23:14:50,236 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e6bdeddabd1140399feab23af51e698e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288160 2024-11-22T23:14:50,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-22T23:14:50,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,242 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:50,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:50,249 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:50,258 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:50,259 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/a4b93a804bd54d28862bdc8ad572e3e1 is 50, key is test_row_0/B:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:50,265 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112216a6c125c46a46fa9ea2357f6664d8d4_f65aabacfa7d7a5aa320670d2d93e1d0 store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:50,272 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112216a6c125c46a46fa9ea2357f6664d8d4_f65aabacfa7d7a5aa320670d2d93e1d0, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:50,272 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112216a6c125c46a46fa9ea2357f6664d8d4_f65aabacfa7d7a5aa320670d2d93e1d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:50,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d60729bdf7184279955f1d5ca6185d17_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317289403/Put/seqid=0 2024-11-22T23:14:50,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741993_1169 (size=12207) 2024-11-22T23:14:50,287 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/a4b93a804bd54d28862bdc8ad572e3e1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/a4b93a804bd54d28862bdc8ad572e3e1 2024-11-22T23:14:50,295 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/B of f65aabacfa7d7a5aa320670d2d93e1d0 into a4b93a804bd54d28862bdc8ad572e3e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:50,295 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:50,295 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/B, priority=13, startTime=1732317290227; duration=0sec 2024-11-22T23:14:50,295 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:50,295 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:B 2024-11-22T23:14:50,295 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:50,296 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:50,296 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/C is initiating minor compaction (all files) 2024-11-22T23:14:50,297 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/C in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:50,297 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/d4e9e08ad8814800bcd9818c2cfa0a0c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.3 K 2024-11-22T23:14:50,297 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e9e08ad8814800bcd9818c2cfa0a0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732317286305 2024-11-22T23:14:50,298 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 68498f61b5b0419db65557b285254af6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732317287003 2024-11-22T23:14:50,298 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e7ab95e5c1924a7680f304f13d031ed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288160 2024-11-22T23:14:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741995_1171 (size=12254) 2024-11-22T23:14:50,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741994_1170 (size=4469) 2024-11-22T23:14:50,318 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:50,319 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/ce961aaa8ed949b5ab179bb507bcb304 is 50, key is test_row_0/C:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:50,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741996_1172 (size=12207) 2024-11-22T23:14:50,329 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/ce961aaa8ed949b5ab179bb507bcb304 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/ce961aaa8ed949b5ab179bb507bcb304 2024-11-22T23:14:50,334 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/C of f65aabacfa7d7a5aa320670d2d93e1d0 into ce961aaa8ed949b5ab179bb507bcb304(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:50,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:50,334 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/C, priority=13, startTime=1732317290227; duration=0sec 2024-11-22T23:14:50,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:50,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:C 2024-11-22T23:14:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:50,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:50,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317350581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,585 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:50,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:50,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317350688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:50,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:50,711 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:50,711 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/80d0b540548045949cf547fbc6a893d1 is 175, key is test_row_0/A:col10/1732317289319/Put/seqid=0 2024-11-22T23:14:50,712 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d60729bdf7184279955f1d5ca6185d17_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d60729bdf7184279955f1d5ca6185d17_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:50,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/4575cc73df8749d3b436f260dd4a35aa, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/4575cc73df8749d3b436f260dd4a35aa is 175, key is test_row_0/A:col10/1732317289403/Put/seqid=0 2024-11-22T23:14:50,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741997_1173 (size=31161) 2024-11-22T23:14:50,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741998_1174 (size=31055) 2024-11-22T23:14:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317350897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:51,125 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/80d0b540548045949cf547fbc6a893d1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1 2024-11-22T23:14:51,130 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/4575cc73df8749d3b436f260dd4a35aa 2024-11-22T23:14:51,132 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/A of f65aabacfa7d7a5aa320670d2d93e1d0 into 80d0b540548045949cf547fbc6a893d1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:51,132 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:51,132 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/A, priority=13, startTime=1732317290227; duration=0sec 2024-11-22T23:14:51,133 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:51,133 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:A 2024-11-22T23:14:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/c98f12d5765b43de82623438c5643676 is 50, key is test_row_0/B:col10/1732317289403/Put/seqid=0 2024-11-22T23:14:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741999_1175 (size=12101) 2024-11-22T23:14:51,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317351198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,213 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317351198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317351213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317351213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,214 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,214 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317351213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,216 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,551 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/c98f12d5765b43de82623438c5643676 2024-11-22T23:14:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e01144ccd80a4728a12fd7d2100572dc is 50, key is test_row_0/C:col10/1732317289403/Put/seqid=0 2024-11-22T23:14:51,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742000_1176 (size=12101) 2024-11-22T23:14:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317351727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:51,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:51,964 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e01144ccd80a4728a12fd7d2100572dc 2024-11-22T23:14:51,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/4575cc73df8749d3b436f260dd4a35aa as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa 2024-11-22T23:14:51,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa, entries=150, sequenceid=131, filesize=30.3 K 2024-11-22T23:14:51,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/c98f12d5765b43de82623438c5643676 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676 2024-11-22T23:14:51,978 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T23:14:51,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/e01144ccd80a4728a12fd7d2100572dc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc 2024-11-22T23:14:51,984 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc, entries=150, sequenceid=131, filesize=11.8 K 2024-11-22T23:14:51,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1744ms, sequenceid=131, compaction requested=false 2024-11-22T23:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-22T23:14:51,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-22T23:14:51,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-22T23:14:51,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-22T23:14:51,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2100 sec 2024-11-22T23:14:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:52,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:52,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:52,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112256b9fb69acde4db6b667449f40c15f20_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:52,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742001_1177 (size=14794) 2024-11-22T23:14:52,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:52,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317352826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:52,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317352937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317353147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:53,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:53,173 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112256b9fb69acde4db6b667449f40c15f20_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112256b9fb69acde4db6b667449f40c15f20_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:53,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/244b157213204d9c9849cde527e659fe, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:53,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/244b157213204d9c9849cde527e659fe is 175, key is test_row_0/A:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742002_1178 (size=39749) 2024-11-22T23:14:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317353458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:53,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/244b157213204d9c9849cde527e659fe 2024-11-22T23:14:53,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/786db38fc68a489eaba5fb68109dd62b is 50, key is test_row_0/B:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742003_1179 (size=12151) 2024-11-22T23:14:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-22T23:14:53,948 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:53,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:53,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-22T23:14:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:14:53,950 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:53,951 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:53,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317353968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:53,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/786db38fc68a489eaba5fb68109dd62b 2024-11-22T23:14:53,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c3bfaa5330d84776abcfd2555d2d2f0a is 50, key is test_row_0/C:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:54,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742004_1180 (size=12151) 2024-11-22T23:14:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:14:54,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:14:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:14:54,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:54,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:14:54,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c3bfaa5330d84776abcfd2555d2d2f0a 2024-11-22T23:14:54,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/244b157213204d9c9849cde527e659fe as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe 2024-11-22T23:14:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:54,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe, entries=200, sequenceid=147, filesize=38.8 K 2024-11-22T23:14:54,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/786db38fc68a489eaba5fb68109dd62b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b 2024-11-22T23:14:54,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b, entries=150, sequenceid=147, filesize=11.9 K 2024-11-22T23:14:54,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c3bfaa5330d84776abcfd2555d2d2f0a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a 2024-11-22T23:14:54,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a, entries=150, sequenceid=147, filesize=11.9 K 2024-11-22T23:14:54,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1709ms, sequenceid=147, compaction requested=true 2024-11-22T23:14:54,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:54,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:54,460 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:54,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:54,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:54,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:54,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:54,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:14:54,463 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:54,465 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:54,465 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/A is initiating minor compaction (all files) 2024-11-22T23:14:54,465 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/A in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,466 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=99.6 K 2024-11-22T23:14:54,466 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,466 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe] 2024-11-22T23:14:54,466 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:54,466 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/B is initiating minor compaction (all files) 2024-11-22T23:14:54,466 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/B in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,467 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/a4b93a804bd54d28862bdc8ad572e3e1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.6 K 2024-11-22T23:14:54,467 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d0b540548045949cf547fbc6a893d1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288160 2024-11-22T23:14:54,467 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4575cc73df8749d3b436f260dd4a35aa, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732317289374 2024-11-22T23:14:54,467 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b93a804bd54d28862bdc8ad572e3e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288160 2024-11-22T23:14:54,468 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 244b157213204d9c9849cde527e659fe, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317290566 2024-11-22T23:14:54,468 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c98f12d5765b43de82623438c5643676, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732317289374 2024-11-22T23:14:54,468 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 786db38fc68a489eaba5fb68109dd62b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317290570 2024-11-22T23:14:54,490 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:54,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/03edba549b9c4708a4f7d8d071d6761a is 50, key is test_row_0/B:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:54,497 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:54,505 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122350ceaf6256b486b891d754f79208154_f65aabacfa7d7a5aa320670d2d93e1d0 store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:54,508 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122350ceaf6256b486b891d754f79208154_f65aabacfa7d7a5aa320670d2d93e1d0, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:54,508 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122350ceaf6256b486b891d754f79208154_f65aabacfa7d7a5aa320670d2d93e1d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:54,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742005_1181 (size=12459) 2024-11-22T23:14:54,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742006_1182 (size=4469) 2024-11-22T23:14:54,540 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#A#compaction#157 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:54,540 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/29950eaa586e483a9aed89d280b50546 is 175, key is test_row_0/A:col10/1732317290579/Put/seqid=0 2024-11-22T23:14:54,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-22T23:14:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,564 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:54,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742007_1183 (size=31413) 2024-11-22T23:14:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:14:54,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228dbc3f192ecb4a10a7817a13fabb9dea_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317292817/Put/seqid=0 2024-11-22T23:14:54,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742008_1184 (size=12304) 2024-11-22T23:14:54,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,655 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228dbc3f192ecb4a10a7817a13fabb9dea_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dbc3f192ecb4a10a7817a13fabb9dea_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/e8b9df2e0783444097a0064e6c40bf56, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/e8b9df2e0783444097a0064e6c40bf56 is 175, key is test_row_0/A:col10/1732317292817/Put/seqid=0 2024-11-22T23:14:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742009_1185 (size=31105) 2024-11-22T23:14:54,685 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/e8b9df2e0783444097a0064e6c40bf56 2024-11-22T23:14:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bba984e694e247b6baf337bc7e9d6e54 is 50, key is test_row_0/B:col10/1732317292817/Put/seqid=0 2024-11-22T23:14:54,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742010_1186 (size=12151) 2024-11-22T23:14:54,734 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bba984e694e247b6baf337bc7e9d6e54 2024-11-22T23:14:54,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/a6063948bab94bc783c2b9aa51a0b157 is 50, key is test_row_0/C:col10/1732317292817/Put/seqid=0 2024-11-22T23:14:54,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742011_1187 (size=12151) 2024-11-22T23:14:54,777 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/a6063948bab94bc783c2b9aa51a0b157 2024-11-22T23:14:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/e8b9df2e0783444097a0064e6c40bf56 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56 2024-11-22T23:14:54,806 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56, entries=150, sequenceid=166, filesize=30.4 K 2024-11-22T23:14:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bba984e694e247b6baf337bc7e9d6e54 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54 2024-11-22T23:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,812 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54, entries=150, sequenceid=166, filesize=11.9 K 2024-11-22T23:14:54,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/a6063948bab94bc783c2b9aa51a0b157 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157 2024-11-22T23:14:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,818 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157, entries=150, sequenceid=166, filesize=11.9 K 2024-11-22T23:14:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,819 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for f65aabacfa7d7a5aa320670d2d93e1d0 in 255ms, sequenceid=166, compaction requested=true 2024-11-22T23:14:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:54,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-22T23:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-22T23:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-22T23:14:54,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-11-22T23:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 876 msec 2024-11-22T23:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,943 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/03edba549b9c4708a4f7d8d071d6761a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/03edba549b9c4708a4f7d8d071d6761a 2024-11-22T23:14:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,954 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/B of f65aabacfa7d7a5aa320670d2d93e1d0 into 03edba549b9c4708a4f7d8d071d6761a(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,954 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:54,954 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/B, priority=13, startTime=1732317294461; duration=0sec 2024-11-22T23:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,954 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:54,954 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:B 2024-11-22T23:14:54,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,954 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,956 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48610 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:14:54,956 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/C is initiating minor compaction (all files) 2024-11-22T23:14:54,956 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/C in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,956 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/ce961aaa8ed949b5ab179bb507bcb304, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=47.5 K 2024-11-22T23:14:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,957 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce961aaa8ed949b5ab179bb507bcb304, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317288160 2024-11-22T23:14:54,957 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01144ccd80a4728a12fd7d2100572dc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732317289374 2024-11-22T23:14:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,958 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3bfaa5330d84776abcfd2555d2d2f0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317290570 2024-11-22T23:14:54,958 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6063948bab94bc783c2b9aa51a0b157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732317292808 2024-11-22T23:14:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,979 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/29950eaa586e483a9aed89d280b50546 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546 2024-11-22T23:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,983 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:54,984 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7e221ac65dd4411f82cf46c6fe558ddb is 50, key is test_row_0/C:col10/1732317292817/Put/seqid=0 2024-11-22T23:14:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,986 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/A of f65aabacfa7d7a5aa320670d2d93e1d0 into 29950eaa586e483a9aed89d280b50546(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:54,986 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:54,986 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/A, priority=13, startTime=1732317294460; duration=0sec 2024-11-22T23:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,986 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:54,986 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:A 2024-11-22T23:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742012_1188 (size=12493) 2024-11-22T23:14:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,017 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7e221ac65dd4411f82cf46c6fe558ddb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e221ac65dd4411f82cf46c6fe558ddb 2024-11-22T23:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,025 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/C of f65aabacfa7d7a5aa320670d2d93e1d0 into 7e221ac65dd4411f82cf46c6fe558ddb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,025 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:55,025 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/C, priority=12, startTime=1732317294461; duration=0sec 2024-11-22T23:14:55,025 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:55,025 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:C 2024-11-22T23:14:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,087 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-22T23:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:55,091 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:55,093 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:55,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:55,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112214cef4d2e6f746bf93c8f6ca96f8eb22_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:55,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742013_1189 (size=19774) 2024-11-22T23:14:55,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317355265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317355267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317295233, max timestamp=1732317295270Num uniques 859; 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317295233, max timestamp=1732317295270Num uniques 859; 2024-11-22T23:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732317295233, max timestamp=1732317295270Num uniques 859; 2024-11-22T23:14:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-22T23:14:55,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317355297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317355306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317355306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317355377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317355377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:55,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317355408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317355417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317355417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317355588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317355588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317355617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317355627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317355627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,647 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:55,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112214cef4d2e6f746bf93c8f6ca96f8eb22_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214cef4d2e6f746bf93c8f6ca96f8eb22_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:55,651 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0f41fdef960940b7871f8e4f3870fc1e, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:55,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0f41fdef960940b7871f8e4f3870fc1e is 175, key is test_row_0/A:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:55,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742014_1190 (size=57033) 2024-11-22T23:14:55,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:55,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:55,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317355897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317355898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317355927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317355947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:55,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,047 INFO [master/a3d87a7361f3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T23:14:56,047 INFO [master/a3d87a7361f3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T23:14:56,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0f41fdef960940b7871f8e4f3870fc1e 2024-11-22T23:14:56,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bdfecd14d1d64d2cb22ccea6a405417b is 50, key is test_row_0/B:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:56,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742015_1191 (size=12151) 2024-11-22T23:14:56,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:56,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317356407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317356407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317356437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317356458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317356458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:56,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:56,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bdfecd14d1d64d2cb22ccea6a405417b 2024-11-22T23:14:56,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/8a257a72a9ee46298cf49a915e4cd9fd is 50, key is test_row_0/C:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:56,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742016_1192 (size=12151) 2024-11-22T23:14:56,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:56,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:14:56,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/8a257a72a9ee46298cf49a915e4cd9fd 2024-11-22T23:14:56,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0f41fdef960940b7871f8e4f3870fc1e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e 2024-11-22T23:14:56,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e, entries=300, sequenceid=179, filesize=55.7 K 2024-11-22T23:14:56,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/bdfecd14d1d64d2cb22ccea6a405417b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b 2024-11-22T23:14:56,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b, entries=150, sequenceid=179, filesize=11.9 K 2024-11-22T23:14:56,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/8a257a72a9ee46298cf49a915e4cd9fd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd 2024-11-22T23:14:56,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd, entries=150, sequenceid=179, filesize=11.9 K 2024-11-22T23:14:56,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1739ms, sequenceid=179, compaction requested=true 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:14:56,908 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:56,908 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:14:56,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:56,909 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:56,909 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:14:56,909 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/B is initiating minor compaction (all files) 2024-11-22T23:14:56,909 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/A is initiating minor compaction (all files) 2024-11-22T23:14:56,909 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/B in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,909 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/A in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,909 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/03edba549b9c4708a4f7d8d071d6761a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=35.9 K 2024-11-22T23:14:56,909 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=116.7 K 2024-11-22T23:14:56,909 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,909 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e] 2024-11-22T23:14:56,910 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29950eaa586e483a9aed89d280b50546, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317290570 2024-11-22T23:14:56,910 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 03edba549b9c4708a4f7d8d071d6761a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317290570 2024-11-22T23:14:56,910 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8b9df2e0783444097a0064e6c40bf56, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732317292808 2024-11-22T23:14:56,910 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bba984e694e247b6baf337bc7e9d6e54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732317292808 2024-11-22T23:14:56,911 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f41fdef960940b7871f8e4f3870fc1e, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732317295009 2024-11-22T23:14:56,911 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfecd14d1d64d2cb22ccea6a405417b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732317295009 2024-11-22T23:14:56,918 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:56,918 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/0d81ef80233c466281a54583f71c788f is 50, key is test_row_0/B:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:56,920 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:56,923 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122152cd89baab047da9ed10b5eb0f23317_f65aabacfa7d7a5aa320670d2d93e1d0 store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:56,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742017_1193 (size=12561) 2024-11-22T23:14:56,925 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122152cd89baab047da9ed10b5eb0f23317_f65aabacfa7d7a5aa320670d2d93e1d0, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:56,926 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122152cd89baab047da9ed10b5eb0f23317_f65aabacfa7d7a5aa320670d2d93e1d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:56,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-22T23:14:56,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:56,929 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:56,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:14:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742018_1194 (size=4469) 2024-11-22T23:14:56,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c4ba19c4385d494d9789c1bfb4314ade_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317295295/Put/seqid=0 2024-11-22T23:14:56,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742019_1195 (size=12304) 2024-11-22T23:14:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:57,329 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/0d81ef80233c466281a54583f71c788f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/0d81ef80233c466281a54583f71c788f 2024-11-22T23:14:57,334 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/B of f65aabacfa7d7a5aa320670d2d93e1d0 into 0d81ef80233c466281a54583f71c788f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:57,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:57,334 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/B, priority=13, startTime=1732317296908; duration=0sec 2024-11-22T23:14:57,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:14:57,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:B 2024-11-22T23:14:57,334 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-22T23:14:57,335 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:14:57,335 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-22T23:14:57,336 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/ad6a3b809f25491195b0ff409edbe516 is 175, key is test_row_0/A:col10/1732317295148/Put/seqid=0 2024-11-22T23:14:57,337 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-22T23:14:57,337 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. because compaction request was cancelled 2024-11-22T23:14:57,337 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:C 2024-11-22T23:14:57,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742020_1196 (size=31515) 2024-11-22T23:14:57,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:57,354 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c4ba19c4385d494d9789c1bfb4314ade_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c4ba19c4385d494d9789c1bfb4314ade_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/f0a84244fa7542148e694884b9d10fc5, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:57,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/f0a84244fa7542148e694884b9d10fc5 is 175, key is test_row_0/A:col10/1732317295295/Put/seqid=0 2024-11-22T23:14:57,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742021_1197 (size=31105) 2024-11-22T23:14:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:57,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317357428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317357429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317357457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,458 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317357467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,469 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317357477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,478 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317357537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317357547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,749 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/ad6a3b809f25491195b0ff409edbe516 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516 2024-11-22T23:14:57,753 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/A of f65aabacfa7d7a5aa320670d2d93e1d0 into ad6a3b809f25491195b0ff409edbe516(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:14:57,753 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:57,753 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/A, priority=13, startTime=1732317296908; duration=0sec 2024-11-22T23:14:57,754 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:14:57,754 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:A 2024-11-22T23:14:57,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317357748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:57,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317357757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:57,766 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/f0a84244fa7542148e694884b9d10fc5 2024-11-22T23:14:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/8cc42162a4eb4ff29794476e6ee03bee is 50, key is test_row_0/B:col10/1732317295295/Put/seqid=0 2024-11-22T23:14:57,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742022_1198 (size=12151) 2024-11-22T23:14:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317358067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:58,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317358067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:58,177 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/8cc42162a4eb4ff29794476e6ee03bee 2024-11-22T23:14:58,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7dd302d739414dfaa51d8ed68b8bc12d is 50, key is test_row_0/C:col10/1732317295295/Put/seqid=0 2024-11-22T23:14:58,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742023_1199 (size=12151) 2024-11-22T23:14:58,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317358577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:58,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317358577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:58,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:58,596 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7dd302d739414dfaa51d8ed68b8bc12d 2024-11-22T23:14:58,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/f0a84244fa7542148e694884b9d10fc5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5 2024-11-22T23:14:58,605 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5, entries=150, sequenceid=215, filesize=30.4 K 2024-11-22T23:14:58,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/8cc42162a4eb4ff29794476e6ee03bee as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee 2024-11-22T23:14:58,610 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee, entries=150, sequenceid=215, filesize=11.9 K 2024-11-22T23:14:58,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/7dd302d739414dfaa51d8ed68b8bc12d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d 2024-11-22T23:14:58,616 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d, entries=150, sequenceid=215, filesize=11.9 K 2024-11-22T23:14:58,617 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1688ms, sequenceid=215, compaction requested=true 2024-11-22T23:14:58,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:14:58,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:58,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-22T23:14:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-22T23:14:58,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-22T23:14:58,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5250 sec 2024-11-22T23:14:58,620 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.5320 sec 2024-11-22T23:14:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-22T23:14:59,278 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:14:59,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:14:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-22T23:14:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:14:59,280 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:14:59,281 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:14:59,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:14:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:14:59,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-22T23:14:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:14:59,433 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:14:59,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:14:59,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229bcf374429e64661b631958ec78f0af1_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317297423/Put/seqid=0 2024-11-22T23:14:59,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742024_1200 (size=12304) 2024-11-22T23:14:59,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:14:59,451 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411229bcf374429e64661b631958ec78f0af1_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229bcf374429e64661b631958ec78f0af1_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/9b91eac044bb42b7aa52ab42b1eb10b2, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:14:59,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/9b91eac044bb42b7aa52ab42b1eb10b2 is 175, key is test_row_0/A:col10/1732317297423/Put/seqid=0 2024-11-22T23:14:59,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742025_1201 (size=31105) 2024-11-22T23:14:59,457 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/9b91eac044bb42b7aa52ab42b1eb10b2 2024-11-22T23:14:59,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/eaae5f90fc194d788325fef1d9dffab9 is 50, key is test_row_0/B:col10/1732317297423/Put/seqid=0 2024-11-22T23:14:59,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742026_1202 (size=12151) 2024-11-22T23:14:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:14:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317299496, max timestamp=1732317299512Num uniques 859; 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317299496, max timestamp=1732317299512Num uniques 859; 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317299496, max timestamp=1732317299512Num uniques 859; 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:59,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:14:59,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317359549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317359553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317359553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317359587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,589 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:14:59,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317359598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,602 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:14:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317359657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317359657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317359668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317359867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317359867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,874 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/eaae5f90fc194d788325fef1d9dffab9 2024-11-22T23:14:59,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/64530e6967c0420fbc0684695bec4899 is 50, key is test_row_0/C:col10/1732317297423/Put/seqid=0 2024-11-22T23:14:59,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:14:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317359877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:14:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:14:59,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:14:59,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742027_1203 (size=12151) 2024-11-22T23:14:59,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:15:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317360177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317360177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317360187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,286 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/64530e6967c0420fbc0684695bec4899 2024-11-22T23:15:00,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/9b91eac044bb42b7aa52ab42b1eb10b2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2 2024-11-22T23:15:00,295 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2, entries=150, sequenceid=226, filesize=30.4 K 2024-11-22T23:15:00,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/eaae5f90fc194d788325fef1d9dffab9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9 2024-11-22T23:15:00,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9, entries=150, sequenceid=226, filesize=11.9 K 2024-11-22T23:15:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/64530e6967c0420fbc0684695bec4899 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899 2024-11-22T23:15:00,305 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899, entries=150, sequenceid=226, filesize=11.9 K 2024-11-22T23:15:00,306 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for f65aabacfa7d7a5aa320670d2d93e1d0 in 873ms, sequenceid=226, compaction requested=true 2024-11-22T23:15:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-22T23:15:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-22T23:15:00,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-22T23:15:00,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0260 sec 2024-11-22T23:15:00,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0310 sec 2024-11-22T23:15:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-22T23:15:00,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:00,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-22T23:15:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:00,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:00,420 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:00,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:00,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-22T23:15:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:00,572 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:15:00,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f249fd7efaa846c3919f182558533f4b_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317299549/Put/seqid=0 2024-11-22T23:15:00,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742028_1204 (size=12454) 2024-11-22T23:15:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:00,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:15:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317360696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317360697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317360699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317360808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317360808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317361017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317361017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,045 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f249fd7efaa846c3919f182558533f4b_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f249fd7efaa846c3919f182558533f4b_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/33827b90f30b47488aa1c00c9362bcd5, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/33827b90f30b47488aa1c00c9362bcd5 is 175, key is test_row_0/A:col10/1732317299549/Put/seqid=0 2024-11-22T23:15:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:01,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742029_1205 (size=31255) 2024-11-22T23:15:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317361327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317361327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,450 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/33827b90f30b47488aa1c00c9362bcd5 2024-11-22T23:15:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/22dce493469141e08c6ace525de61502 is 50, key is test_row_0/B:col10/1732317299549/Put/seqid=0 2024-11-22T23:15:01,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742030_1206 (size=12301) 2024-11-22T23:15:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:01,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317361607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,610 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317361628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,630 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317361717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,720 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317361837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317361838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:01,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:01,862 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/22dce493469141e08c6ace525de61502 2024-11-22T23:15:01,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/3416b02a7c0b4d1eacfe80a062fb5717 is 50, key is test_row_0/C:col10/1732317299549/Put/seqid=0 2024-11-22T23:15:01,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742031_1207 (size=12301) 2024-11-22T23:15:02,275 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/3416b02a7c0b4d1eacfe80a062fb5717 2024-11-22T23:15:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/33827b90f30b47488aa1c00c9362bcd5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5 2024-11-22T23:15:02,283 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5, entries=150, sequenceid=262, filesize=30.5 K 2024-11-22T23:15:02,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/22dce493469141e08c6ace525de61502 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502 2024-11-22T23:15:02,288 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502, entries=150, sequenceid=262, filesize=12.0 K 2024-11-22T23:15:02,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/3416b02a7c0b4d1eacfe80a062fb5717 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717 2024-11-22T23:15:02,293 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717, entries=150, sequenceid=262, filesize=12.0 K 2024-11-22T23:15:02,294 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1722ms, sequenceid=262, compaction requested=true 2024-11-22T23:15:02,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:02,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:02,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-22T23:15:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-22T23:15:02,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-22T23:15:02,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8760 sec 2024-11-22T23:15:02,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.8800 sec 2024-11-22T23:15:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-22T23:15:02,568 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:02,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-22T23:15:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:02,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:02,571 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:02,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:02,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-22T23:15:02,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:02,724 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:15:02,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:02,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a408f454fc24820943ec19e37729172_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317300692/Put/seqid=0 2024-11-22T23:15:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742032_1208 (size=12454) 2024-11-22T23:15:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:02,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:15:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:02,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317302851, max timestamp=1732317302891Num uniques 859; 2024-11-22T23:15:02,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:02,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317302851, max timestamp=1732317302891Num uniques 859; 2024-11-22T23:15:02,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317302851, max timestamp=1732317302891Num uniques 859; 2024-11-22T23:15:02,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:02,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317362968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317362971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317363077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317363077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:03,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228a408f454fc24820943ec19e37729172_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a408f454fc24820943ec19e37729172_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e55e23cddbe470b8937182bb496612a, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:03,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e55e23cddbe470b8937182bb496612a is 175, key is test_row_0/A:col10/1732317300692/Put/seqid=0 2024-11-22T23:15:03,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742033_1209 (size=31255) 2024-11-22T23:15:03,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:03,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317363287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317363287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,551 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e55e23cddbe470b8937182bb496612a 2024-11-22T23:15:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fc727be2155e4ef0af9f7a9be1cee602 is 50, key is test_row_0/B:col10/1732317300692/Put/seqid=0 2024-11-22T23:15:03,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742034_1210 (size=12301) 2024-11-22T23:15:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317363597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317363598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:03,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37586 deadline: 1732317363728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:03,732 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:03,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:03,963 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fc727be2155e4ef0af9f7a9be1cee602 2024-11-22T23:15:03,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/82183101d85b40788747820fb755183c is 50, key is test_row_0/C:col10/1732317300692/Put/seqid=0 2024-11-22T23:15:03,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742035_1211 (size=12301) 2024-11-22T23:15:04,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317364107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:04,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:04,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317364118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:04,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:04,374 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/82183101d85b40788747820fb755183c 2024-11-22T23:15:04,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e55e23cddbe470b8937182bb496612a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a 2024-11-22T23:15:04,382 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a, entries=150, sequenceid=270, filesize=30.5 K 2024-11-22T23:15:04,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/fc727be2155e4ef0af9f7a9be1cee602 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602 2024-11-22T23:15:04,387 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T23:15:04,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/82183101d85b40788747820fb755183c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c 2024-11-22T23:15:04,392 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c, entries=150, sequenceid=270, filesize=12.0 K 2024-11-22T23:15:04,393 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1669ms, sequenceid=270, compaction requested=true 2024-11-22T23:15:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-22T23:15:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-22T23:15:04,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-22T23:15:04,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-22T23:15:04,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.8280 sec 2024-11-22T23:15:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-22T23:15:04,728 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:04,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-22T23:15:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:04,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:04,730 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:04,731 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:04,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-22T23:15:04,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:04,883 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:15:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:04,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224d016cb8429f4759b980898795e28777_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317302911/Put/seqid=0 2024-11-22T23:15:04,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742036_1212 (size=12454) 2024-11-22T23:15:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:05,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. as already flushing 2024-11-22T23:15:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317365139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317365140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317365247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317365258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:05,299 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224d016cb8429f4759b980898795e28777_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224d016cb8429f4759b980898795e28777_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:05,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e33ef44a9b84758adcf333feed50fb4, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:05,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e33ef44a9b84758adcf333feed50fb4 is 175, key is test_row_0/A:col10/1732317302911/Put/seqid=0 2024-11-22T23:15:05,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742037_1213 (size=31255) 2024-11-22T23:15:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:05,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317365457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317365467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,489 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-22T23:15:05,489 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:05,489 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,489 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:05,495 INFO [Thread-716 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-716. 2024-11-22T23:15:05,495 DEBUG [Thread-716 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:05,495 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,495 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:05,495 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-22T23:15:05,495 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:05,495 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,495 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:05,502 INFO [Thread-722 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-722. 2024-11-22T23:15:05,502 DEBUG [Thread-722 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:05,502 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,502 DEBUG [Thread-722 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:05,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37528 deadline: 1732317365617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,619 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37570 deadline: 1732317365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,650 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in f65aabacfa7d7a5aa320670d2d93e1d0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,705 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e33ef44a9b84758adcf333feed50fb4 2024-11-22T23:15:05,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/14a0201ad4354818b514179290d39721 is 50, key is test_row_0/B:col10/1732317302911/Put/seqid=0 2024-11-22T23:15:05,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742038_1214 (size=12301) 2024-11-22T23:15:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317365768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317365788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:06,122 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/14a0201ad4354818b514179290d39721 2024-11-22T23:15:06,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/fddc7f6e1f6042b1a0357f7c54c3a73c is 50, key is test_row_0/C:col10/1732317302911/Put/seqid=0 2024-11-22T23:15:06,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742039_1215 (size=12301) 2024-11-22T23:15:06,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732317366278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:06,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:06,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:06,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37540 deadline: 1732317366297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:06,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/fddc7f6e1f6042b1a0357f7c54c3a73c 2024-11-22T23:15:06,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/0e33ef44a9b84758adcf333feed50fb4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4 2024-11-22T23:15:06,610 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4, entries=150, sequenceid=307, filesize=30.5 K 2024-11-22T23:15:06,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/14a0201ad4354818b514179290d39721 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721 2024-11-22T23:15:06,615 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721, entries=150, sequenceid=307, filesize=12.0 K 2024-11-22T23:15:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/fddc7f6e1f6042b1a0357f7c54c3a73c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c 2024-11-22T23:15:06,620 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c, entries=150, sequenceid=307, filesize=12.0 K 2024-11-22T23:15:06,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1738ms, sequenceid=307, compaction requested=true 2024-11-22T23:15:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:06,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-22T23:15:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-22T23:15:06,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-22T23:15:06,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-22T23:15:06,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.8970 sec 2024-11-22T23:15:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-22T23:15:06,879 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:07,301 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-22T23:15:07,302 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:07,302 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,302 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:07,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:07,312 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-22T23:15:07,312 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:15:07,312 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:15:07,312 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:15:07,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:07,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d3d14d17532645acb2703eae9ff703e7_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:07,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742040_1216 (size=12454) 2024-11-22T23:15:07,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:07,735 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d3d14d17532645acb2703eae9ff703e7_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3d14d17532645acb2703eae9ff703e7_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:07,736 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/2431c0e835a5417590ced2f4c9286a94, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:07,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/2431c0e835a5417590ced2f4c9286a94 is 175, key is test_row_0/A:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:07,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742041_1217 (size=31255) 2024-11-22T23:15:07,759 INFO [Thread-705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-705. 2024-11-22T23:15:07,759 DEBUG [Thread-705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:07,759 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,759 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:07,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:08,141 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/2431c0e835a5417590ced2f4c9286a94 2024-11-22T23:15:08,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/4ea0d927923942299150e19fc7e985d1 is 50, key is test_row_0/B:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:08,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742042_1218 (size=12301) 2024-11-22T23:15:08,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/4ea0d927923942299150e19fc7e985d1 2024-11-22T23:15:08,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c98f33823ada4c6b8edc4be45bf4f893 is 50, key is test_row_0/C:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:08,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742043_1219 (size=12301) 2024-11-22T23:15:08,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c98f33823ada4c6b8edc4be45bf4f893 2024-11-22T23:15:08,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/2431c0e835a5417590ced2f4c9286a94 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94 2024-11-22T23:15:08,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94, entries=150, sequenceid=317, filesize=30.5 K 2024-11-22T23:15:08,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/4ea0d927923942299150e19fc7e985d1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1 2024-11-22T23:15:09,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T23:15:09,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/c98f33823ada4c6b8edc4be45bf4f893 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893 2024-11-22T23:15:09,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893, entries=150, sequenceid=317, filesize=12.0 K 2024-11-22T23:15:09,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1694ms, sequenceid=317, compaction requested=true 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:09,005 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f65aabacfa7d7a5aa320670d2d93e1d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:09,005 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-22T23:15:09,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:09,007 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 218745 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-22T23:15:09,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 86067 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-22T23:15:09,007 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/A is initiating minor compaction (all files) 2024-11-22T23:15:09,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/B is initiating minor compaction (all files) 2024-11-22T23:15:09,007 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/A in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:09,007 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/B in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:09,007 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=213.6 K 2024-11-22T23:15:09,007 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/0d81ef80233c466281a54583f71c788f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=84.0 K 2024-11-22T23:15:09,007 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:09,007 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94] 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d81ef80233c466281a54583f71c788f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732317295009 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad6a3b809f25491195b0ff409edbe516, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732317295009 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0a84244fa7542148e694884b9d10fc5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732317295291 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc42162a4eb4ff29794476e6ee03bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732317295291 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b91eac044bb42b7aa52ab42b1eb10b2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732317297423 2024-11-22T23:15:09,008 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting eaae5f90fc194d788325fef1d9dffab9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732317297423 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33827b90f30b47488aa1c00c9362bcd5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732317299543 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 22dce493469141e08c6ace525de61502, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732317299543 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e55e23cddbe470b8937182bb496612a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732317300692 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fc727be2155e4ef0af9f7a9be1cee602, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732317300692 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e33ef44a9b84758adcf333feed50fb4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317302911 2024-11-22T23:15:09,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a0201ad4354818b514179290d39721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317302911 2024-11-22T23:15:09,010 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2431c0e835a5417590ced2f4c9286a94, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732317305131 2024-11-22T23:15:09,010 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ea0d927923942299150e19fc7e985d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732317305131 2024-11-22T23:15:09,025 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:09,025 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#B#compaction#185 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:09,026 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/405a69e7a0ed4244b8e679e13f0ad2a5 is 50, key is test_row_0/B:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:09,030 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112265507821d3774223a5edbd082a774e76_f65aabacfa7d7a5aa320670d2d93e1d0 store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:09,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742044_1220 (size=12949) 2024-11-22T23:15:09,070 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112265507821d3774223a5edbd082a774e76_f65aabacfa7d7a5aa320670d2d93e1d0, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:09,070 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112265507821d3774223a5edbd082a774e76_f65aabacfa7d7a5aa320670d2d93e1d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742045_1221 (size=4469) 2024-11-22T23:15:09,444 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/405a69e7a0ed4244b8e679e13f0ad2a5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/405a69e7a0ed4244b8e679e13f0ad2a5 2024-11-22T23:15:09,449 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/B of f65aabacfa7d7a5aa320670d2d93e1d0 into 405a69e7a0ed4244b8e679e13f0ad2a5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:09,449 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:09,449 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/B, priority=9, startTime=1732317309005; duration=0sec 2024-11-22T23:15:09,449 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:09,449 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:B 2024-11-22T23:15:09,449 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-22T23:15:09,452 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 98150 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-22T23:15:09,452 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): f65aabacfa7d7a5aa320670d2d93e1d0/C is initiating minor compaction (all files) 2024-11-22T23:15:09,452 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f65aabacfa7d7a5aa320670d2d93e1d0/C in TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:09,452 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e221ac65dd4411f82cf46c6fe558ddb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp, totalSize=95.8 K 2024-11-22T23:15:09,452 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e221ac65dd4411f82cf46c6fe558ddb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732317292808 2024-11-22T23:15:09,453 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a257a72a9ee46298cf49a915e4cd9fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1732317295009 2024-11-22T23:15:09,453 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd302d739414dfaa51d8ed68b8bc12d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732317295291 2024-11-22T23:15:09,453 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 64530e6967c0420fbc0684695bec4899, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732317297423 2024-11-22T23:15:09,453 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3416b02a7c0b4d1eacfe80a062fb5717, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1732317299543 2024-11-22T23:15:09,454 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 82183101d85b40788747820fb755183c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732317300692 2024-11-22T23:15:09,454 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fddc7f6e1f6042b1a0357f7c54c3a73c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317302911 2024-11-22T23:15:09,454 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c98f33823ada4c6b8edc4be45bf4f893, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732317305131 2024-11-22T23:15:09,472 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#C#compaction#187 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:09,472 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/36705333539f4c47b1a72f23b82c7274 is 50, key is test_row_0/C:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:09,475 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f65aabacfa7d7a5aa320670d2d93e1d0#A#compaction#186 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:09,475 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/a2aa4c161ffb492fa84a6986bafc691c is 175, key is test_row_0/A:col10/1732317305131/Put/seqid=0 2024-11-22T23:15:09,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742046_1222 (size=12915) 2024-11-22T23:15:09,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742047_1223 (size=31903) 2024-11-22T23:15:09,481 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/36705333539f4c47b1a72f23b82c7274 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/36705333539f4c47b1a72f23b82c7274 2024-11-22T23:15:09,487 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/C of f65aabacfa7d7a5aa320670d2d93e1d0 into 36705333539f4c47b1a72f23b82c7274(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:09,487 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:09,487 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/C, priority=8, startTime=1732317309005; duration=0sec 2024-11-22T23:15:09,487 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:09,487 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:C 2024-11-22T23:15:09,807 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:15:09,892 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/a2aa4c161ffb492fa84a6986bafc691c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/a2aa4c161ffb492fa84a6986bafc691c 2024-11-22T23:15:09,898 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in f65aabacfa7d7a5aa320670d2d93e1d0/A of f65aabacfa7d7a5aa320670d2d93e1d0 into a2aa4c161ffb492fa84a6986bafc691c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:09,898 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f65aabacfa7d7a5aa320670d2d93e1d0: 2024-11-22T23:15:09,898 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., storeName=f65aabacfa7d7a5aa320670d2d93e1d0/A, priority=9, startTime=1732317309005; duration=0sec 2024-11-22T23:15:09,898 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:09,898 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f65aabacfa7d7a5aa320670d2d93e1d0:A 2024-11-22T23:15:15,681 INFO [Thread-713 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-713. 2024-11-22T23:15:15,681 DEBUG [Thread-713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:15,681 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,681 DEBUG [Thread-713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,681 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:15,709 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-22T23:15:15,709 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:15,709 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,709 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2976 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2830 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1244 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3732 rows 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1242 2024-11-22T23:15:15,710 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3722 rows 2024-11-22T23:15:15,710 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:15:15,710 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:15:15,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:15,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:15,711 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:15:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:15,714 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317315714"}]},"ts":"1732317315714"} 2024-11-22T23:15:15,716 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:15:15,716 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:15:15,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:15:15,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, UNASSIGN}] 2024-11-22T23:15:15,719 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, UNASSIGN 2024-11-22T23:15:15,720 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:15,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, UNASSIGN because future has completed 2024-11-22T23:15:15,722 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:15:15,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:15,875 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing f65aabacfa7d7a5aa320670d2d93e1d0, disabling compactions & flushes 2024-11-22T23:15:15,875 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. after waiting 0 ms 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:15,875 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing f65aabacfa7d7a5aa320670d2d93e1d0 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-22T23:15:15,875 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=A 2024-11-22T23:15:15,876 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:15,876 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=B 2024-11-22T23:15:15,876 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:15,876 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f65aabacfa7d7a5aa320670d2d93e1d0, store=C 2024-11-22T23:15:15,876 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:15,881 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112291ae675597b44bd0b89aa5cacf987947_f65aabacfa7d7a5aa320670d2d93e1d0 is 50, key is test_row_0/A:col10/1732317307758/Put/seqid=0 2024-11-22T23:15:15,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742048_1224 (size=12454) 2024-11-22T23:15:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:16,287 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:16,297 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112291ae675597b44bd0b89aa5cacf987947_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291ae675597b44bd0b89aa5cacf987947_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:16,299 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/c85c6410030e4d8aae76e29d1245e604, store: [table=TestAcidGuarantees family=A region=f65aabacfa7d7a5aa320670d2d93e1d0] 2024-11-22T23:15:16,300 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/c85c6410030e4d8aae76e29d1245e604 is 175, key is test_row_0/A:col10/1732317307758/Put/seqid=0 2024-11-22T23:15:16,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742049_1225 (size=31255) 2024-11-22T23:15:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:16,704 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/c85c6410030e4d8aae76e29d1245e604 2024-11-22T23:15:16,712 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/b7fc99c08a2c4969bcf47784daccd2d1 is 50, key is test_row_0/B:col10/1732317307758/Put/seqid=0 2024-11-22T23:15:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742050_1226 (size=12301) 2024-11-22T23:15:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:17,116 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/b7fc99c08a2c4969bcf47784daccd2d1 2024-11-22T23:15:17,125 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/681d9bc8d8da4095936186a068068165 is 50, key is test_row_0/C:col10/1732317307758/Put/seqid=0 2024-11-22T23:15:17,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742051_1227 (size=12301) 2024-11-22T23:15:17,530 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/681d9bc8d8da4095936186a068068165 2024-11-22T23:15:17,536 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/A/c85c6410030e4d8aae76e29d1245e604 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/c85c6410030e4d8aae76e29d1245e604 2024-11-22T23:15:17,541 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/c85c6410030e4d8aae76e29d1245e604, entries=150, sequenceid=326, filesize=30.5 K 2024-11-22T23:15:17,542 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/B/b7fc99c08a2c4969bcf47784daccd2d1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/b7fc99c08a2c4969bcf47784daccd2d1 2024-11-22T23:15:17,547 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/b7fc99c08a2c4969bcf47784daccd2d1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-22T23:15:17,548 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/.tmp/C/681d9bc8d8da4095936186a068068165 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/681d9bc8d8da4095936186a068068165 2024-11-22T23:15:17,552 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/681d9bc8d8da4095936186a068068165, entries=150, sequenceid=326, filesize=12.0 K 2024-11-22T23:15:17,553 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1678ms, sequenceid=326, compaction requested=false 2024-11-22T23:15:17,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94] to archive 2024-11-22T23:15:17,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:17,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d137919e9cbc4320bf99cae4d972126c 2024-11-22T23:15:17,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0707bcc289a54701901e942303ec250c 2024-11-22T23:15:17,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/cbc377e3c8e94739b4fce8fbc974870d 2024-11-22T23:15:17,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/d25c56865fe4439e9a8a70d5cbc90fa9 2024-11-22T23:15:17,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/6cdc17d2fe7b42489ec5b5e0885682c2 2024-11-22T23:15:17,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/04158921d697443ea72c8a2ba2cd3cdb 2024-11-22T23:15:17,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/80d0b540548045949cf547fbc6a893d1 2024-11-22T23:15:17,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/4575cc73df8749d3b436f260dd4a35aa 2024-11-22T23:15:17,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/244b157213204d9c9849cde527e659fe 2024-11-22T23:15:17,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/29950eaa586e483a9aed89d280b50546 2024-11-22T23:15:17,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/e8b9df2e0783444097a0064e6c40bf56 2024-11-22T23:15:17,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0f41fdef960940b7871f8e4f3870fc1e 2024-11-22T23:15:17,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/ad6a3b809f25491195b0ff409edbe516 2024-11-22T23:15:17,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/f0a84244fa7542148e694884b9d10fc5 2024-11-22T23:15:17,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/9b91eac044bb42b7aa52ab42b1eb10b2 2024-11-22T23:15:17,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/33827b90f30b47488aa1c00c9362bcd5 2024-11-22T23:15:17,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e55e23cddbe470b8937182bb496612a 2024-11-22T23:15:17,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/0e33ef44a9b84758adcf333feed50fb4 2024-11-22T23:15:17,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/2431c0e835a5417590ced2f4c9286a94 2024-11-22T23:15:17,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/328243e7bce84b31ad77eb722e294d65, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/a4b93a804bd54d28862bdc8ad572e3e1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/03edba549b9c4708a4f7d8d071d6761a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/0d81ef80233c466281a54583f71c788f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1] to archive 2024-11-22T23:15:17,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:17,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/85df224d3f6b4238be269e6db0502b16 2024-11-22T23:15:17,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/ff2aa86ed0b348839d6428d49b4a89db 2024-11-22T23:15:17,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/328243e7bce84b31ad77eb722e294d65 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/328243e7bce84b31ad77eb722e294d65 2024-11-22T23:15:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e47d683a13f840afb9bcd005d92e4607 2024-11-22T23:15:17,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fa72855165594312b9a7fc79f3f0bd53 2024-11-22T23:15:17,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/a4b93a804bd54d28862bdc8ad572e3e1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/a4b93a804bd54d28862bdc8ad572e3e1 2024-11-22T23:15:17,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/e6bdeddabd1140399feab23af51e698e 2024-11-22T23:15:17,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/c98f12d5765b43de82623438c5643676 2024-11-22T23:15:17,619 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/03edba549b9c4708a4f7d8d071d6761a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/03edba549b9c4708a4f7d8d071d6761a 2024-11-22T23:15:17,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/786db38fc68a489eaba5fb68109dd62b 2024-11-22T23:15:17,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bba984e694e247b6baf337bc7e9d6e54 2024-11-22T23:15:17,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/0d81ef80233c466281a54583f71c788f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/0d81ef80233c466281a54583f71c788f 2024-11-22T23:15:17,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/bdfecd14d1d64d2cb22ccea6a405417b 2024-11-22T23:15:17,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/8cc42162a4eb4ff29794476e6ee03bee 2024-11-22T23:15:17,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/eaae5f90fc194d788325fef1d9dffab9 2024-11-22T23:15:17,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/22dce493469141e08c6ace525de61502 2024-11-22T23:15:17,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/fc727be2155e4ef0af9f7a9be1cee602 2024-11-22T23:15:17,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/14a0201ad4354818b514179290d39721 2024-11-22T23:15:17,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/4ea0d927923942299150e19fc7e985d1 2024-11-22T23:15:17,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/d4e9e08ad8814800bcd9818c2cfa0a0c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/ce961aaa8ed949b5ab179bb507bcb304, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e221ac65dd4411f82cf46c6fe558ddb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893] to archive 2024-11-22T23:15:17,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:17,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/66d2e9052862418fafa19a8a5cca5e7d 2024-11-22T23:15:17,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e1416036121497b855b6213ae56889d 2024-11-22T23:15:17,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/d4e9e08ad8814800bcd9818c2cfa0a0c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/d4e9e08ad8814800bcd9818c2cfa0a0c 2024-11-22T23:15:17,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/2166f1e76d0e4e95a5efffd94eb4540a 2024-11-22T23:15:17,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/68498f61b5b0419db65557b285254af6 2024-11-22T23:15:17,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/ce961aaa8ed949b5ab179bb507bcb304 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/ce961aaa8ed949b5ab179bb507bcb304 2024-11-22T23:15:17,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e7ab95e5c1924a7680f304f13d031ed7 2024-11-22T23:15:17,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/e01144ccd80a4728a12fd7d2100572dc 2024-11-22T23:15:17,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c3bfaa5330d84776abcfd2555d2d2f0a 2024-11-22T23:15:17,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e221ac65dd4411f82cf46c6fe558ddb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7e221ac65dd4411f82cf46c6fe558ddb 2024-11-22T23:15:17,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/a6063948bab94bc783c2b9aa51a0b157 2024-11-22T23:15:17,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/8a257a72a9ee46298cf49a915e4cd9fd 2024-11-22T23:15:17,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/7dd302d739414dfaa51d8ed68b8bc12d 2024-11-22T23:15:17,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/64530e6967c0420fbc0684695bec4899 2024-11-22T23:15:17,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/3416b02a7c0b4d1eacfe80a062fb5717 2024-11-22T23:15:17,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/82183101d85b40788747820fb755183c 2024-11-22T23:15:17,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/fddc7f6e1f6042b1a0357f7c54c3a73c 2024-11-22T23:15:17,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/c98f33823ada4c6b8edc4be45bf4f893 2024-11-22T23:15:17,680 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits/329.seqid, newMaxSeqId=329, maxSeqId=4 2024-11-22T23:15:17,681 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. 2024-11-22T23:15:17,682 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for f65aabacfa7d7a5aa320670d2d93e1d0: Waiting for close lock at 1732317315875Running coprocessor pre-close hooks at 1732317315875Disabling compacts and flushes for region at 1732317315875Disabling writes for close at 1732317315875Obtaining lock to block concurrent updates at 1732317315875Preparing flush snapshotting stores in f65aabacfa7d7a5aa320670d2d93e1d0 at 1732317315875Finished memstore snapshotting TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732317315876 (+1 ms)Flushing stores of TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0. at 1732317315876Flushing f65aabacfa7d7a5aa320670d2d93e1d0/A: creating writer at 1732317315876Flushing mob file f65aabacfa7d7a5aa320670d2d93e1d0/A: appending metadata at 1732317315881 (+5 ms)Flushing mob file f65aabacfa7d7a5aa320670d2d93e1d0/A: closing flushed file at 1732317315881Flushing f65aabacfa7d7a5aa320670d2d93e1d0/A: appending metadata at 1732317316299 (+418 ms)Flushing f65aabacfa7d7a5aa320670d2d93e1d0/A: closing flushed file at 1732317316299Flushing f65aabacfa7d7a5aa320670d2d93e1d0/B: creating writer at 1732317316708 (+409 ms)Flushing f65aabacfa7d7a5aa320670d2d93e1d0/B: appending metadata at 1732317316711 (+3 ms)Flushing f65aabacfa7d7a5aa320670d2d93e1d0/B: closing flushed file at 1732317316711Flushing f65aabacfa7d7a5aa320670d2d93e1d0/C: creating writer at 1732317317121 (+410 ms)Flushing f65aabacfa7d7a5aa320670d2d93e1d0/C: appending metadata at 1732317317125 (+4 ms)Flushing f65aabacfa7d7a5aa320670d2d93e1d0/C: closing flushed file at 1732317317125Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47c3c6e1: reopening flushed file at 1732317317535 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d1f8d6c: reopening flushed file at 1732317317541 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ccae672: reopening flushed file at 1732317317547 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for f65aabacfa7d7a5aa320670d2d93e1d0 in 1678ms, sequenceid=326, compaction requested=false at 1732317317553 (+6 ms)Writing region close event to WAL at 1732317317676 (+123 ms)Running coprocessor post-close hooks at 1732317317681 (+5 ms)Closed at 1732317317681 2024-11-22T23:15:17,684 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,685 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=f65aabacfa7d7a5aa320670d2d93e1d0, regionState=CLOSED 2024-11-22T23:15:17,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:17,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-22T23:15:17,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure f65aabacfa7d7a5aa320670d2d93e1d0, server=a3d87a7361f3,46715,1732317251834 in 1.9660 sec 2024-11-22T23:15:17,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-22T23:15:17,694 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f65aabacfa7d7a5aa320670d2d93e1d0, UNASSIGN in 1.9730 sec 2024-11-22T23:15:17,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-22T23:15:17,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9780 sec 2024-11-22T23:15:17,702 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317317702"}]},"ts":"1732317317702"} 2024-11-22T23:15:17,704 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:15:17,704 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:15:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9940 sec 2024-11-22T23:15:17,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-22T23:15:17,868 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:17,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:15:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,870 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T23:15:17,871 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,874 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,879 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits] 2024-11-22T23:15:17,886 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/a2aa4c161ffb492fa84a6986bafc691c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/a2aa4c161ffb492fa84a6986bafc691c 2024-11-22T23:15:17,887 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/c85c6410030e4d8aae76e29d1245e604 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/A/c85c6410030e4d8aae76e29d1245e604 2024-11-22T23:15:17,891 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/405a69e7a0ed4244b8e679e13f0ad2a5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/405a69e7a0ed4244b8e679e13f0ad2a5 2024-11-22T23:15:17,893 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/b7fc99c08a2c4969bcf47784daccd2d1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/B/b7fc99c08a2c4969bcf47784daccd2d1 2024-11-22T23:15:17,895 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/36705333539f4c47b1a72f23b82c7274 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/36705333539f4c47b1a72f23b82c7274 2024-11-22T23:15:17,896 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/681d9bc8d8da4095936186a068068165 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/C/681d9bc8d8da4095936186a068068165 2024-11-22T23:15:17,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits/329.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0/recovered.edits/329.seqid 2024-11-22T23:15:17,899 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,899 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:15:17,900 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:15:17,901 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:15:17,905 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214cef4d2e6f746bf93c8f6ca96f8eb22_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112214cef4d2e6f746bf93c8f6ca96f8eb22_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,906 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224d016cb8429f4759b980898795e28777_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224d016cb8429f4759b980898795e28777_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,907 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112256b9fb69acde4db6b667449f40c15f20_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112256b9fb69acde4db6b667449f40c15f20_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,908 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226ac155afb53e409aba9d0268c9900462_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226ac155afb53e409aba9d0268c9900462_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,910 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a408f454fc24820943ec19e37729172_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a408f454fc24820943ec19e37729172_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,911 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a6a55cbf7d44993831b1e22b96ee08e_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228a6a55cbf7d44993831b1e22b96ee08e_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dbc3f192ecb4a10a7817a13fabb9dea_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dbc3f192ecb4a10a7817a13fabb9dea_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,915 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291ae675597b44bd0b89aa5cacf987947_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112291ae675597b44bd0b89aa5cacf987947_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,916 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229bcf374429e64661b631958ec78f0af1_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411229bcf374429e64661b631958ec78f0af1_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,918 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c4ba19c4385d494d9789c1bfb4314ade_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c4ba19c4385d494d9789c1bfb4314ade_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,920 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d333fbb8cb5046cc86e6b1e8ab00a908_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d333fbb8cb5046cc86e6b1e8ab00a908_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,921 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3d14d17532645acb2703eae9ff703e7_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d3d14d17532645acb2703eae9ff703e7_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,923 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d60729bdf7184279955f1d5ca6185d17_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122d60729bdf7184279955f1d5ca6185d17_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,925 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122df33558563de45c68797f0f030773a55_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122df33558563de45c68797f0f030773a55_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,926 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f249fd7efaa846c3919f182558533f4b_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f249fd7efaa846c3919f182558533f4b_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,928 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f71be6f267e5490e9cb61c9540433b0a_f65aabacfa7d7a5aa320670d2d93e1d0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f71be6f267e5490e9cb61c9540433b0a_f65aabacfa7d7a5aa320670d2d93e1d0 2024-11-22T23:15:17,929 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:15:17,932 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,935 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:15:17,938 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:15:17,939 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,939 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:15:17,940 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317317939"}]},"ts":"9223372036854775807"} 2024-11-22T23:15:17,942 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:15:17,942 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f65aabacfa7d7a5aa320670d2d93e1d0, NAME => 'TestAcidGuarantees,,1732317283291.f65aabacfa7d7a5aa320670d2d93e1d0.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:15:17,942 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:15:17,943 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317317942"}]},"ts":"9223372036854775807"} 2024-11-22T23:15:17,945 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:15:17,946 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:17,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-22T23:15:17,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-22T23:15:17,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:15:17,978 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:17,989 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=344 (was 388), ProcessCount=11 (was 11), AvailableMemoryMB=2029 (was 856) - AvailableMemoryMB LEAK? - 2024-11-22T23:15:18,002 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=344, ProcessCount=11, AvailableMemoryMB=2027 2024-11-22T23:15:18,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:15:18,003 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:15:18,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:18,005 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:15:18,005 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:18,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-22T23:15:18,006 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:15:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742052_1228 (size=1061) 2024-11-22T23:15:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:18,415 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:15:18,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742053_1229 (size=53) 2024-11-22T23:15:18,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 72b8054778e76f5d48c7bea222a97b27, disabling compactions & flushes 2024-11-22T23:15:18,823 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. after waiting 0 ms 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:18,823 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:18,823 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72b8054778e76f5d48c7bea222a97b27: Waiting for close lock at 1732317318823Disabling compacts and flushes for region at 1732317318823Disabling writes for close at 1732317318823Writing region close event to WAL at 1732317318823Closed at 1732317318823 2024-11-22T23:15:18,825 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:15:18,826 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317318826"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317318826"}]},"ts":"1732317318826"} 2024-11-22T23:15:18,829 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:15:18,831 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:15:18,831 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317318831"}]},"ts":"1732317318831"} 2024-11-22T23:15:18,833 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:15:18,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, ASSIGN}] 2024-11-22T23:15:18,834 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, ASSIGN 2024-11-22T23:15:18,834 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:15:18,985 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=72b8054778e76f5d48c7bea222a97b27, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:18,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, ASSIGN because future has completed 2024-11-22T23:15:18,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:19,143 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:19,144 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:15:19,144 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,144 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:15:19,144 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,144 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,145 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,146 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:19,146 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b8054778e76f5d48c7bea222a97b27 columnFamilyName A 2024-11-22T23:15:19,146 DEBUG [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:19,147 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(327): Store=72b8054778e76f5d48c7bea222a97b27/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:19,147 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:19,148 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:19,148 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b8054778e76f5d48c7bea222a97b27 columnFamilyName B 2024-11-22T23:15:19,148 DEBUG [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:19,149 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(327): Store=72b8054778e76f5d48c7bea222a97b27/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:19,149 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,150 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:19,150 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72b8054778e76f5d48c7bea222a97b27 columnFamilyName C 2024-11-22T23:15:19,150 DEBUG [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:19,151 INFO [StoreOpener-72b8054778e76f5d48c7bea222a97b27-1 {}] regionserver.HStore(327): Store=72b8054778e76f5d48c7bea222a97b27/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:19,151 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:19,151 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,151 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,152 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,152 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,152 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,153 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:15:19,153 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,155 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:15:19,156 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 72b8054778e76f5d48c7bea222a97b27; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58894769, jitterRate=-0.12239955365657806}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:15:19,156 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:19,157 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 72b8054778e76f5d48c7bea222a97b27: Running coprocessor pre-open hook at 1732317319144Writing region info on filesystem at 1732317319144Initializing all the Stores at 1732317319145 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317319145Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317319145Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317319145Cleaning up temporary data from old regions at 1732317319152 (+7 ms)Running coprocessor post-open hooks at 1732317319156 (+4 ms)Region opened successfully at 1732317319157 (+1 ms) 2024-11-22T23:15:19,158 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., pid=69, masterSystemTime=1732317319141 2024-11-22T23:15:19,160 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:19,160 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:19,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=72b8054778e76f5d48c7bea222a97b27, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:19,162 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:19,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-22T23:15:19,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 in 175 msec 2024-11-22T23:15:19,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-22T23:15:19,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, ASSIGN in 331 msec 2024-11-22T23:15:19,166 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:15:19,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317319166"}]},"ts":"1732317319166"} 2024-11-22T23:15:19,168 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:15:19,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:15:19,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-22T23:15:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-22T23:15:20,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:20,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4b6d1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,170 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f94bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,172 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,175 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e0f0acc 2024-11-22T23:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:15:20,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:15:20,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,179 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbf7439, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6697beb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,183 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3492aedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,184 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,185 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7f6880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,186 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,188 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ebc3df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,189 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708336ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d8ec240, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7382749c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,197 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15fdf5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,199 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184b4373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,201 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4516d581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,203 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62f3125f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463f7dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,206 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,209 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5c3c57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,210 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63455081, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,212 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ea6e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8acb8f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:20,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:20,216 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:20,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:20,217 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:20,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ed3b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:20,218 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,218 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,218 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,219 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,219 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,220 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,221 DEBUG [Thread-1090 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:20,221 DEBUG [Thread-1090 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,223 DEBUG [Thread-1075 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,224 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-22T23:15:20,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:20,230 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:20,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:20,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:20,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,233 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,234 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:20,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:20,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:15:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:20,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:20,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:20,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585812d9220844bbb568589114ea7c53 is 50, key is test_row_0/A:col10/1732317320229/Put/seqid=0 2024-11-22T23:15:20,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742054_1230 (size=12001) 2024-11-22T23:15:20,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317320301, max timestamp=1732317320312Num uniques 859; 2024-11-22T23:15:20,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:20,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317320301, max timestamp=1732317320312Num uniques 859; 2024-11-22T23:15:20,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:20,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732317320301, max timestamp=1732317320312Num uniques 859; 2024-11-22T23:15:20,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7050, suffix heap size=117064,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=154 2024-11-22T23:15:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:20,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317380337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317380338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732317320335, max timestamp=1732317320347Num uniques 1018; 2024-11-22T23:15:20,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-22T23:15:20,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317320335, max timestamp=1732317320347Num uniques 1018; 2024-11-22T23:15:20,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-22T23:15:20,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732317320333, max timestamp=1732317320347Num uniques 1013; 2024-11-22T23:15:20,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46393, new segment data size=12268, suffix heap size=133320,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=268 2024-11-22T23:15:20,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317380390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317380457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317380458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317380507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317380507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317380509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317380667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317380667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585812d9220844bbb568589114ea7c53 2024-11-22T23:15:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317380717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317380718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317380718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0ac64ea7e6cd47a7a54d18ea03da2371 is 50, key is test_row_0/B:col10/1732317320229/Put/seqid=0 2024-11-22T23:15:20,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742055_1231 (size=12001) 2024-11-22T23:15:20,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0ac64ea7e6cd47a7a54d18ea03da2371 2024-11-22T23:15:20,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:20,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:20,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a62da71308af49248f315bac8b781b80 is 50, key is test_row_0/C:col10/1732317320229/Put/seqid=0 2024-11-22T23:15:20,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742056_1232 (size=12001) 2024-11-22T23:15:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317380978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317380978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317381038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317381038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317381038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a62da71308af49248f315bac8b781b80 2024-11-22T23:15:21,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585812d9220844bbb568589114ea7c53 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53 2024-11-22T23:15:21,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53, entries=150, sequenceid=11, filesize=11.7 K 2024-11-22T23:15:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0ac64ea7e6cd47a7a54d18ea03da2371 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371 2024-11-22T23:15:21,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371, entries=150, sequenceid=11, filesize=11.7 K 2024-11-22T23:15:21,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a62da71308af49248f315bac8b781b80 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80 2024-11-22T23:15:21,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:21,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80, entries=150, sequenceid=11, filesize=11.7 K 2024-11-22T23:15:21,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.38 KB/136582 for 72b8054778e76f5d48c7bea222a97b27 in 1046ms, sequenceid=11, compaction requested=false 2024-11-22T23:15:21,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:21,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:21,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-22T23:15:21,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:21,467 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=133.38 KB heapSize=377.63 KB 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:21,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:21,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/93a46d3c4bd249579990cfdf0e1773c5 is 50, key is test_row_0/A:col10/1732317320391/Put/seqid=0 2024-11-22T23:15:21,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742057_1233 (size=12001) 2024-11-22T23:15:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:21,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:21,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317381501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317381504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317381548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317381548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317381548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317381608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317381618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317381818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317381827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:21,879 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/93a46d3c4bd249579990cfdf0e1773c5 2024-11-22T23:15:21,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/6b6d20b7ab004679b4dfc53493baab1e is 50, key is test_row_0/B:col10/1732317320391/Put/seqid=0 2024-11-22T23:15:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742058_1234 (size=12001) 2024-11-22T23:15:21,918 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/6b6d20b7ab004679b4dfc53493baab1e 2024-11-22T23:15:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a2982c7f3a3c4aefb0f5eac445bd453e is 50, key is test_row_0/C:col10/1732317320391/Put/seqid=0 2024-11-22T23:15:21,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742059_1235 (size=12001) 2024-11-22T23:15:22,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317382127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317382137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,348 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a2982c7f3a3c4aefb0f5eac445bd453e 2024-11-22T23:15:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/93a46d3c4bd249579990cfdf0e1773c5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5 2024-11-22T23:15:22,357 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5, entries=150, sequenceid=64, filesize=11.7 K 2024-11-22T23:15:22,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/6b6d20b7ab004679b4dfc53493baab1e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e 2024-11-22T23:15:22,362 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e, entries=150, sequenceid=64, filesize=11.7 K 2024-11-22T23:15:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a2982c7f3a3c4aefb0f5eac445bd453e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e 2024-11-22T23:15:22,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e, entries=150, sequenceid=64, filesize=11.7 K 2024-11-22T23:15:22,367 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~133.38 KB/136582, heapSize ~377.58 KB/386640, currentSize=46.96 KB/48090 for 72b8054778e76f5d48c7bea222a97b27 in 900ms, sequenceid=64, compaction requested=false 2024-11-22T23:15:22,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:22,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:22,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-22T23:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-22T23:15:22,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-22T23:15:22,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1390 sec 2024-11-22T23:15:22,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1500 sec 2024-11-22T23:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-22T23:15:22,378 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:22,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-22T23:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:15:22,385 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:22,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:22,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:15:22,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-22T23:15:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:22,540 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:22,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:22,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4ac4a7cab61946f3ae1d4504eb5bee1c is 50, key is test_row_0/A:col10/1732317321503/Put/seqid=0 2024-11-22T23:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:22,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:22,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742060_1236 (size=12001) 2024-11-22T23:15:22,597 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317322582, max timestamp=1732317322595Num uniques 859; 2024-11-22T23:15:22,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:22,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317322582, max timestamp=1732317322595Num uniques 859; 2024-11-22T23:15:22,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:22,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732317322582, max timestamp=1732317322595Num uniques 859; 2024-11-22T23:15:22,598 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:22,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317382637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317382647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317382668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317382668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317382669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:15:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317382788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317382788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317382788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:22,965 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4ac4a7cab61946f3ae1d4504eb5bee1c 2024-11-22T23:15:22,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/2997517ddf25489fad9126e7f230a8c6 is 50, key is test_row_0/B:col10/1732317321503/Put/seqid=0 2024-11-22T23:15:22,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742061_1237 (size=12001) 2024-11-22T23:15:22,997 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/2997517ddf25489fad9126e7f230a8c6 2024-11-22T23:15:23,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317382998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317382998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317382998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:15:23,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/88b6683b04b24a2987f55a40e71ce0d6 is 50, key is test_row_0/C:col10/1732317321503/Put/seqid=0 2024-11-22T23:15:23,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742062_1238 (size=12001) 2024-11-22T23:15:23,209 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:15:23,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317383307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317383308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317383308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,413 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/88b6683b04b24a2987f55a40e71ce0d6 2024-11-22T23:15:23,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4ac4a7cab61946f3ae1d4504eb5bee1c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c 2024-11-22T23:15:23,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T23:15:23,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/2997517ddf25489fad9126e7f230a8c6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6 2024-11-22T23:15:23,426 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T23:15:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/88b6683b04b24a2987f55a40e71ce0d6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6 2024-11-22T23:15:23,431 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6, entries=150, sequenceid=74, filesize=11.7 K 2024-11-22T23:15:23,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 72b8054778e76f5d48c7bea222a97b27 in 892ms, sequenceid=74, compaction requested=true 2024-11-22T23:15:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:23,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-22T23:15:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-22T23:15:23,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-22T23:15:23,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0480 sec 2024-11-22T23:15:23,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0600 sec 2024-11-22T23:15:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-22T23:15:23,518 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:23,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-22T23:15:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:15:23,520 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:23,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:23,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:23,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:15:23,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:23,673 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7c4babe2369245f5a40e071a254c959d is 50, key is test_row_0/A:col10/1732317322637/Put/seqid=0 2024-11-22T23:15:23,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:23,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:23,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742063_1239 (size=12001) 2024-11-22T23:15:23,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7c4babe2369245f5a40e071a254c959d 2024-11-22T23:15:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/aeebbe6a25024f8fa6d16e9f269f4010 is 50, key is test_row_0/B:col10/1732317322637/Put/seqid=0 2024-11-22T23:15:23,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742064_1240 (size=12001) 2024-11-22T23:15:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317383700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317383702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317383817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317383817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317383817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317383818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317383828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:15:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317384027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317384027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,106 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/aeebbe6a25024f8fa6d16e9f269f4010 2024-11-22T23:15:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b96998453709460f821a902b55b362ce is 50, key is test_row_0/C:col10/1732317322637/Put/seqid=0 2024-11-22T23:15:24,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742065_1241 (size=12001) 2024-11-22T23:15:24,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:15:24,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317384338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317384338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,527 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b96998453709460f821a902b55b362ce 2024-11-22T23:15:24,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7c4babe2369245f5a40e071a254c959d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d 2024-11-22T23:15:24,535 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d, entries=150, sequenceid=111, filesize=11.7 K 2024-11-22T23:15:24,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/aeebbe6a25024f8fa6d16e9f269f4010 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010 2024-11-22T23:15:24,539 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010, entries=150, sequenceid=111, filesize=11.7 K 2024-11-22T23:15:24,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b96998453709460f821a902b55b362ce as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce 2024-11-22T23:15:24,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce, entries=150, sequenceid=111, filesize=11.7 K 2024-11-22T23:15:24,545 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 72b8054778e76f5d48c7bea222a97b27 in 872ms, sequenceid=111, compaction requested=true 2024-11-22T23:15:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-22T23:15:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-22T23:15:24,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-22T23:15:24,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-22T23:15:24,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0300 sec 2024-11-22T23:15:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-22T23:15:24,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:24,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-22T23:15:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:15:24,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:24,660 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:24,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:15:24,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:24,812 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:24,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/732c0b308da64ab89430be1569b53111 is 50, key is test_row_0/A:col10/1732317323692/Put/seqid=0 2024-11-22T23:15:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:24,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:24,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742066_1242 (size=12001) 2024-11-22T23:15:24,842 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/732c0b308da64ab89430be1569b53111 2024-11-22T23:15:24,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9f96b456f58c4868b3f7e3e6e506216d is 50, key is test_row_0/B:col10/1732317323692/Put/seqid=0 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317324863, max timestamp=1732317324873Num uniques 859; 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317324863, max timestamp=1732317324873Num uniques 859; 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317324863, max timestamp=1732317324873Num uniques 859; 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:24,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742067_1243 (size=12001) 2024-11-22T23:15:24,890 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9f96b456f58c4868b3f7e3e6e506216d 2024-11-22T23:15:24,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cf4059aa6c734119b9b96b32edf56c73 is 50, key is test_row_0/C:col10/1732317323692/Put/seqid=0 2024-11-22T23:15:24,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317384905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317384905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742068_1244 (size=12001) 2024-11-22T23:15:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cf4059aa6c734119b9b96b32edf56c73 2024-11-22T23:15:24,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317384912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317384912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317384913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/732c0b308da64ab89430be1569b53111 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111 2024-11-22T23:15:24,928 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:15:24,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9f96b456f58c4868b3f7e3e6e506216d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d 2024-11-22T23:15:24,935 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:15:24,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cf4059aa6c734119b9b96b32edf56c73 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73 2024-11-22T23:15:24,942 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73, entries=150, sequenceid=120, filesize=11.7 K 2024-11-22T23:15:24,944 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 72b8054778e76f5d48c7bea222a97b27 in 132ms, sequenceid=120, compaction requested=true 2024-11-22T23:15:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:24,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-22T23:15:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-22T23:15:24,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-22T23:15:24,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 286 msec 2024-11-22T23:15:24,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 291 msec 2024-11-22T23:15:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-22T23:15:24,978 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:24,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-22T23:15:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:24,982 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:24,984 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:24,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:25,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-22T23:15:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:25,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317324901, max timestamp=1732317325020Num uniques 1068; 2024-11-22T23:15:25,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-22T23:15:25,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317324901, max timestamp=1732317325020Num uniques 1068; 2024-11-22T23:15:25,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-22T23:15:25,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317324901, max timestamp=1732317325020Num uniques 1068; 2024-11-22T23:15:25,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:25,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585fae0c13b74a2bb25fd50d0783fec3 is 50, key is test_row_0/A:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:25,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742069_1245 (size=14541) 2024-11-22T23:15:25,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585fae0c13b74a2bb25fd50d0783fec3 2024-11-22T23:15:25,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/fadd83627198459b8823d467556621dd is 50, key is test_row_0/B:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:25,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742070_1246 (size=12151) 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732317325054, max timestamp=1732317325066Num uniques 859; 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732317325054, max timestamp=1732317325066Num uniques 859; 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732317325054, max timestamp=1732317325066Num uniques 859; 2024-11-22T23:15:25,070 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:15:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317385109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317385110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317385110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317385111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317325094, max timestamp=1732317325118Num uniques 1118; 2024-11-22T23:15:25,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-22T23:15:25,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317325094, max timestamp=1732317325118Num uniques 1118; 2024-11-22T23:15:25,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-22T23:15:25,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317325094, max timestamp=1732317325118Num uniques 1118; 2024-11-22T23:15:25,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:25,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14968, totalHeapSize=34248, min timestamp=1732317325198, max timestamp=1732317325240Num uniques 1027; 2024-11-22T23:15:25,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47028, new segment data size=14968, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-22T23:15:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732317325198, max timestamp=1732317325240Num uniques 1027; 2024-11-22T23:15:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-22T23:15:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14971, totalHeapSize=34248, min timestamp=1732317325198, max timestamp=1732317325240Num uniques 1027; 2024-11-22T23:15:25,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47031, new segment data size=14971, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-22T23:15:25,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317325275, max timestamp=1732317325279Num uniques 1186; 2024-11-22T23:15:25,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-11-22T23:15:25,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317325275, max timestamp=1732317325279Num uniques 1186; 2024-11-22T23:15:25,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317325275, max timestamp=1732317325279Num uniques 1186; 2024-11-22T23:15:25,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-11-22T23:15:25,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=8516, suffix heap size=151312,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=186 2024-11-22T23:15:25,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:25,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732317325347, max timestamp=1732317325356Num uniques 1045; 2024-11-22T23:15:25,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732317325347, max timestamp=1732317325356Num uniques 1045; 2024-11-22T23:15:25,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-22T23:15:25,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-22T23:15:25,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732317325347, max timestamp=1732317325356Num uniques 1045; 2024-11-22T23:15:25,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-22T23:15:25,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317385383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732317325377, max timestamp=1732317325388Num uniques 1104; 2024-11-22T23:15:25,392 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-22T23:15:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732317325377, max timestamp=1732317325388Num uniques 1104; 2024-11-22T23:15:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-22T23:15:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1732317325377, max timestamp=1732317325388Num uniques 1104; 2024-11-22T23:15:25,393 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-22T23:15:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317385448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317385449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/fadd83627198459b8823d467556621dd 2024-11-22T23:15:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317385459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317385460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/514d96c9daf34debac617d5f3f8b054b is 50, key is test_row_0/C:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:25,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742071_1247 (size=12151) 2024-11-22T23:15:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317385497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317385567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317385567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317385577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317385577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:25,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317385707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317385777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317385777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317385787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317385787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:25,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:25,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.38 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/514d96c9daf34debac617d5f3f8b054b 2024-11-22T23:15:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/585fae0c13b74a2bb25fd50d0783fec3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3 2024-11-22T23:15:25,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3, entries=200, sequenceid=159, filesize=14.2 K 2024-11-22T23:15:25,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/fadd83627198459b8823d467556621dd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd 2024-11-22T23:15:25,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T23:15:25,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/514d96c9daf34debac617d5f3f8b054b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b 2024-11-22T23:15:25,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-22T23:15:25,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~31.95 KB/32714, heapSize ~81.61 KB/83568, currentSize=154.31 KB/158010 for 72b8054778e76f5d48c7bea222a97b27 in 872ms, sequenceid=159, compaction requested=true 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:25,893 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:25,893 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:25,895 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72156 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:15:25,895 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74546 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:25,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,896 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=70.5 K 2024-11-22T23:15:25,896 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=72.8 K 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585812d9220844bbb568589114ea7c53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732317320229 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac64ea7e6cd47a7a54d18ea03da2371, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732317320229 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6d20b7ab004679b4dfc53493baab1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732317320383 2024-11-22T23:15:25,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a46d3c4bd249579990cfdf0e1773c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732317320383 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2997517ddf25489fad9126e7f230a8c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732317321496 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ac4a7cab61946f3ae1d4504eb5bee1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732317321496 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting aeebbe6a25024f8fa6d16e9f269f4010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317322628 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c4babe2369245f5a40e071a254c959d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317322628 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f96b456f58c4868b3f7e3e6e506216d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732317323692 2024-11-22T23:15:25,897 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732c0b308da64ab89430be1569b53111, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732317323692 2024-11-22T23:15:25,898 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585fae0c13b74a2bb25fd50d0783fec3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:25,898 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fadd83627198459b8823d467556621dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:25,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:25,909 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=154.31 KB heapSize=431.16 KB 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:25,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:25,910 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#209 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:25,911 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/dbf3465fe9184f6f9a1e30e5a64fe4c6 is 50, key is test_row_0/B:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:25,914 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#210 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:25,914 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/f86e709676e24d77b13f7f035dab52f2 is 50, key is test_row_0/A:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4c767a5650394dff8dd6be441718931d is 50, key is test_row_0/A:col10/1732317325458/Put/seqid=0 2024-11-22T23:15:25,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742072_1248 (size=12357) 2024-11-22T23:15:25,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742073_1249 (size=12357) 2024-11-22T23:15:25,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742074_1250 (size=12301) 2024-11-22T23:15:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4c767a5650394dff8dd6be441718931d 2024-11-22T23:15:25,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f0d8e1d1ab244a64879575425775f207 is 50, key is test_row_0/B:col10/1732317325458/Put/seqid=0 2024-11-22T23:15:25,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742075_1251 (size=12301) 2024-11-22T23:15:25,955 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f0d8e1d1ab244a64879575425775f207 2024-11-22T23:15:25,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/6c8b0fe93d0d4021aaf5fa3590f654ac is 50, key is test_row_0/C:col10/1732317325458/Put/seqid=0 2024-11-22T23:15:25,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742076_1252 (size=12301) 2024-11-22T23:15:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:26,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317386053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317386087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317386088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317386097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317386097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:26,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317386168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,330 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/dbf3465fe9184f6f9a1e30e5a64fe4c6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/dbf3465fe9184f6f9a1e30e5a64fe4c6 2024-11-22T23:15:26,341 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into dbf3465fe9184f6f9a1e30e5a64fe4c6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:26,341 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:26,341 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=10, startTime=1732317325893; duration=0sec 2024-11-22T23:15:26,341 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:26,341 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:26,341 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-22T23:15:26,343 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/f86e709676e24d77b13f7f035dab52f2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/f86e709676e24d77b13f7f035dab52f2 2024-11-22T23:15:26,344 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72156 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-22T23:15:26,344 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:26,344 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:26,344 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=70.5 K 2024-11-22T23:15:26,344 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a62da71308af49248f315bac8b781b80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732317320229 2024-11-22T23:15:26,345 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a2982c7f3a3c4aefb0f5eac445bd453e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1732317320383 2024-11-22T23:15:26,346 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 88b6683b04b24a2987f55a40e71ce0d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732317321496 2024-11-22T23:15:26,347 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b96998453709460f821a902b55b362ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732317322628 2024-11-22T23:15:26,347 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting cf4059aa6c734119b9b96b32edf56c73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732317323692 2024-11-22T23:15:26,348 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 514d96c9daf34debac617d5f3f8b054b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:26,352 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into f86e709676e24d77b13f7f035dab52f2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:26,352 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:26,352 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=10, startTime=1732317325893; duration=0sec 2024-11-22T23:15:26,352 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:26,352 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:26,359 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#214 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:26,360 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/1196fca6bcca446086787d018e51908f is 50, key is test_row_0/C:col10/1732317325021/Put/seqid=0 2024-11-22T23:15:26,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317386377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,387 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/6c8b0fe93d0d4021aaf5fa3590f654ac 2024-11-22T23:15:26,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742077_1253 (size=12357) 2024-11-22T23:15:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4c767a5650394dff8dd6be441718931d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d 2024-11-22T23:15:26,401 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d, entries=150, sequenceid=282, filesize=12.0 K 2024-11-22T23:15:26,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f0d8e1d1ab244a64879575425775f207 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207 2024-11-22T23:15:26,408 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207, entries=150, sequenceid=282, filesize=12.0 K 2024-11-22T23:15:26,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/6c8b0fe93d0d4021aaf5fa3590f654ac as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac 2024-11-22T23:15:26,414 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac, entries=150, sequenceid=282, filesize=12.0 K 2024-11-22T23:15:26,415 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.11 KB/441456, currentSize=33.54 KB/34350 for 72b8054778e76f5d48c7bea222a97b27 in 506ms, sequenceid=282, compaction requested=false 2024-11-22T23:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:26,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-22T23:15:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-22T23:15:26,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-22T23:15:26,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4330 sec 2024-11-22T23:15:26,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4480 sec 2024-11-22T23:15:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:26,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:26,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:26,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:26,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:26,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:26,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:26,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:26,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/fb1f4e9912854ee2a01ef7147eaf2af2 is 50, key is test_row_0/A:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:26,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742078_1254 (size=14741) 2024-11-22T23:15:26,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317326635, max timestamp=1732317326647Num uniques 859; 2024-11-22T23:15:26,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:26,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317326635, max timestamp=1732317326647Num uniques 859; 2024-11-22T23:15:26,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:26,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317326635, max timestamp=1732317326647Num uniques 859; 2024-11-22T23:15:26,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317386687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317386691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317386694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317386694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317386695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,799 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/1196fca6bcca446086787d018e51908f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/1196fca6bcca446086787d018e51908f 2024-11-22T23:15:26,804 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into 1196fca6bcca446086787d018e51908f(size=12.1 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:26,804 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:26,804 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=10, startTime=1732317325893; duration=0sec 2024-11-22T23:15:26,804 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:26,804 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:26,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317386807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317386807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317386807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317386808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:26,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317387017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317387017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317387017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317387017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/fb1f4e9912854ee2a01ef7147eaf2af2 2024-11-22T23:15:27,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/ee1f7fbaed9142d3b83a26a156cfa22c is 50, key is test_row_0/B:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:27,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742079_1255 (size=12301) 2024-11-22T23:15:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-22T23:15:27,128 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:27,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-22T23:15:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:27,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:27,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:27,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:27,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317387208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:27,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:15:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317387328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317387328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317387328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317387328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/ee1f7fbaed9142d3b83a26a156cfa22c 2024-11-22T23:15:27,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/71a8865ed24244b9994da5db0e25542a is 50, key is test_row_0/C:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:27,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742080_1256 (size=12301) 2024-11-22T23:15:27,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:15:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:15:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317387838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317387838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317387838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317387847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:27,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:27,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/71a8865ed24244b9994da5db0e25542a 2024-11-22T23:15:27,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/fb1f4e9912854ee2a01ef7147eaf2af2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2 2024-11-22T23:15:27,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2, entries=200, sequenceid=294, filesize=14.4 K 2024-11-22T23:15:27,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/ee1f7fbaed9142d3b83a26a156cfa22c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c 2024-11-22T23:15:27,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c, entries=150, sequenceid=294, filesize=12.0 K 2024-11-22T23:15:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/71a8865ed24244b9994da5db0e25542a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a 2024-11-22T23:15:27,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a, entries=150, sequenceid=294, filesize=12.0 K 2024-11-22T23:15:27,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 72b8054778e76f5d48c7bea222a97b27 in 1275ms, sequenceid=294, compaction requested=true 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:27,877 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:27,877 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:27,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:27,878 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:27,878 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:27,878 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:27,878 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:27,878 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,878 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,878 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/dbf3465fe9184f6f9a1e30e5a64fe4c6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.1 K 2024-11-22T23:15:27,878 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/f86e709676e24d77b13f7f035dab52f2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=38.5 K 2024-11-22T23:15:27,879 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f86e709676e24d77b13f7f035dab52f2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:27,879 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf3465fe9184f6f9a1e30e5a64fe4c6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:27,880 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f0d8e1d1ab244a64879575425775f207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732317325419 2024-11-22T23:15:27,880 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c767a5650394dff8dd6be441718931d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732317325419 2024-11-22T23:15:27,880 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1f7fbaed9142d3b83a26a156cfa22c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326051 2024-11-22T23:15:27,880 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb1f4e9912854ee2a01ef7147eaf2af2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326021 2024-11-22T23:15:27,888 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:27,889 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#219 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:27,889 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/44135002df294bd2bfc14caf0783b9d5 is 50, key is test_row_0/A:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:27,889 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/6df0428ecb8d4df1b011e217e2212bf3 is 50, key is test_row_0/B:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:27,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,898 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:27,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:27,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742081_1257 (size=12609) 2024-11-22T23:15:27,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742082_1258 (size=12609) 2024-11-22T23:15:27,916 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/6df0428ecb8d4df1b011e217e2212bf3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6df0428ecb8d4df1b011e217e2212bf3 2024-11-22T23:15:27,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7655bb68b87c47768e03a7e035112a16 is 50, key is test_row_0/A:col10/1732317326692/Put/seqid=0 2024-11-22T23:15:27,920 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into 6df0428ecb8d4df1b011e217e2212bf3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:27,920 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:27,920 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317327877; duration=0sec 2024-11-22T23:15:27,920 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:27,920 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:27,920 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:27,923 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:27,923 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:27,923 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:27,923 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/1196fca6bcca446086787d018e51908f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.1 K 2024-11-22T23:15:27,926 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1196fca6bcca446086787d018e51908f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732317324910 2024-11-22T23:15:27,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742083_1259 (size=12301) 2024-11-22T23:15:27,927 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8b0fe93d0d4021aaf5fa3590f654ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732317325419 2024-11-22T23:15:27,927 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7655bb68b87c47768e03a7e035112a16 2024-11-22T23:15:27,927 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a8865ed24244b9994da5db0e25542a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326051 2024-11-22T23:15:27,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/da6bca73bd6d48ef83f29384b577151d is 50, key is test_row_0/B:col10/1732317326692/Put/seqid=0 2024-11-22T23:15:27,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742084_1260 (size=12301) 2024-11-22T23:15:27,956 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:27,957 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/9f68aae388914b01957f43cdecd00583 is 50, key is test_row_0/C:col10/1732317326597/Put/seqid=0 2024-11-22T23:15:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742085_1261 (size=12609) 2024-11-22T23:15:27,971 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/9f68aae388914b01957f43cdecd00583 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/9f68aae388914b01957f43cdecd00583 2024-11-22T23:15:27,982 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into 9f68aae388914b01957f43cdecd00583(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:27,982 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:27,982 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317327877; duration=0sec 2024-11-22T23:15:27,983 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:27,983 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:28,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317388275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,318 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/44135002df294bd2bfc14caf0783b9d5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44135002df294bd2bfc14caf0783b9d5 2024-11-22T23:15:28,324 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 44135002df294bd2bfc14caf0783b9d5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:28,324 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:28,324 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317327877; duration=0sec 2024-11-22T23:15:28,324 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:28,324 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:28,350 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/da6bca73bd6d48ef83f29384b577151d 2024-11-22T23:15:28,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0b62fd5d3f5647feb98a1d79fcb58349 is 50, key is test_row_0/C:col10/1732317326692/Put/seqid=0 2024-11-22T23:15:28,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742086_1262 (size=12301) 2024-11-22T23:15:28,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317388387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317388598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,771 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0b62fd5d3f5647feb98a1d79fcb58349 2024-11-22T23:15:28,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/7655bb68b87c47768e03a7e035112a16 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16 2024-11-22T23:15:28,783 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16, entries=150, sequenceid=332, filesize=12.0 K 2024-11-22T23:15:28,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/da6bca73bd6d48ef83f29384b577151d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d 2024-11-22T23:15:28,791 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-22T23:15:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0b62fd5d3f5647feb98a1d79fcb58349 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349 2024-11-22T23:15:28,805 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349, entries=150, sequenceid=332, filesize=12.0 K 2024-11-22T23:15:28,807 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 72b8054778e76f5d48c7bea222a97b27 in 909ms, sequenceid=332, compaction requested=false 2024-11-22T23:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:28,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-22T23:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-22T23:15:28,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-22T23:15:28,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-22T23:15:28,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6850 sec 2024-11-22T23:15:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:28,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:28,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:28,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4d0a7f63df7740cb9d9437350d9184f5 is 50, key is test_row_0/A:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:28,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742087_1263 (size=14741) 2024-11-22T23:15:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317388879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317388879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317388880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317388883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317388907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317388988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317388988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317388988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:28,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317388997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:28,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317389198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317389198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317389198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317389207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4d0a7f63df7740cb9d9437350d9184f5 2024-11-22T23:15:29,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5d325dc04f174d05b8657152d5b91f3e is 50, key is test_row_0/B:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-22T23:15:29,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:29,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-22T23:15:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:29,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:29,280 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:29,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:29,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742088_1264 (size=12301) 2024-11-22T23:15:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:29,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317389417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:15:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:29,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317389507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317389507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317389507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317389517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:29,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:15:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:29,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5d325dc04f174d05b8657152d5b91f3e 2024-11-22T23:15:29,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/8bd4f522ad6c4683a1761a8ed33c44e1 is 50, key is test_row_0/C:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:29,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742089_1265 (size=12301) 2024-11-22T23:15:29,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/8bd4f522ad6c4683a1761a8ed33c44e1 2024-11-22T23:15:29,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/4d0a7f63df7740cb9d9437350d9184f5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5 2024-11-22T23:15:29,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5, entries=200, sequenceid=347, filesize=14.4 K 2024-11-22T23:15:29,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5d325dc04f174d05b8657152d5b91f3e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e 2024-11-22T23:15:29,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e, entries=150, sequenceid=347, filesize=12.0 K 2024-11-22T23:15:29,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/8bd4f522ad6c4683a1761a8ed33c44e1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1 2024-11-22T23:15:29,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1, entries=150, sequenceid=347, filesize=12.0 K 2024-11-22T23:15:29,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 72b8054778e76f5d48c7bea222a97b27 in 886ms, sequenceid=347, compaction requested=true 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:29,736 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:29,736 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:29,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:29,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,738 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:29,738 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:29,738 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:29,738 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:29,738 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:29,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:29,738 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,738 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:29,738 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44135002df294bd2bfc14caf0783b9d5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=38.7 K 2024-11-22T23:15:29,738 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6df0428ecb8d4df1b011e217e2212bf3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.3 K 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df0428ecb8d4df1b011e217e2212bf3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326051 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44135002df294bd2bfc14caf0783b9d5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326051 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting da6bca73bd6d48ef83f29384b577151d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732317326672 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7655bb68b87c47768e03a7e035112a16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732317326672 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d325dc04f174d05b8657152d5b91f3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:29,739 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d0a7f63df7740cb9d9437350d9184f5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:29,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/6bb32a2e4e3543459073104203fd2d93 is 50, key is test_row_0/A:col10/1732317328877/Put/seqid=0 2024-11-22T23:15:29,750 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:29,751 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/468f3c2197914208ba0e109adfce1cc5 is 50, key is test_row_0/A:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:29,752 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:29,752 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0c969bb0203746acb0723675f27a704b is 50, key is test_row_0/B:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742092_1268 (size=12711) 2024-11-22T23:15:29,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742091_1267 (size=12711) 2024-11-22T23:15:29,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742090_1266 (size=12301) 2024-11-22T23:15:29,761 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/6bb32a2e4e3543459073104203fd2d93 2024-11-22T23:15:29,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5ae941e8f41d4594b50c3eedc5d76e27 is 50, key is test_row_0/B:col10/1732317328877/Put/seqid=0 2024-11-22T23:15:29,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742093_1269 (size=12301) 2024-11-22T23:15:29,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:30,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:30,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317390057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317390057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317390057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317390058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,161 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0c969bb0203746acb0723675f27a704b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0c969bb0203746acb0723675f27a704b 2024-11-22T23:15:30,162 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/468f3c2197914208ba0e109adfce1cc5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/468f3c2197914208ba0e109adfce1cc5 2024-11-22T23:15:30,166 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 468f3c2197914208ba0e109adfce1cc5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:30,166 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into 0c969bb0203746acb0723675f27a704b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:30,167 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317329736; duration=0sec 2024-11-22T23:15:30,167 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317329736; duration=0sec 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:30,167 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:30,168 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:30,168 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:30,168 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:30,169 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/9f68aae388914b01957f43cdecd00583, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.3 K 2024-11-22T23:15:30,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317390167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,169 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f68aae388914b01957f43cdecd00583, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1732317326051 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317390167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317390167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,170 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b62fd5d3f5647feb98a1d79fcb58349, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732317326672 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317390167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,170 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bd4f522ad6c4683a1761a8ed33c44e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,178 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:30,179 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5ae941e8f41d4594b50c3eedc5d76e27 2024-11-22T23:15:30,179 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/be8d9070293a44e29879443d8adbf10a is 50, key is test_row_0/C:col10/1732317328274/Put/seqid=0 2024-11-22T23:15:30,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742094_1270 (size=12711) 2024-11-22T23:15:30,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ec08ea2223394ec8b0bfd688942a6043 is 50, key is test_row_0/C:col10/1732317328877/Put/seqid=0 2024-11-22T23:15:30,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742095_1271 (size=12301) 2024-11-22T23:15:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317390377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317390377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317390377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317390377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:30,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317390427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,430 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,590 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/be8d9070293a44e29879443d8adbf10a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/be8d9070293a44e29879443d8adbf10a 2024-11-22T23:15:30,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ec08ea2223394ec8b0bfd688942a6043 2024-11-22T23:15:30,595 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into be8d9070293a44e29879443d8adbf10a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:30,595 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:30,595 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317329736; duration=0sec 2024-11-22T23:15:30,595 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:30,595 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/6bb32a2e4e3543459073104203fd2d93 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93 2024-11-22T23:15:30,602 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93, entries=150, sequenceid=367, filesize=12.0 K 2024-11-22T23:15:30,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5ae941e8f41d4594b50c3eedc5d76e27 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27 2024-11-22T23:15:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27, entries=150, sequenceid=367, filesize=12.0 K 2024-11-22T23:15:30,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ec08ea2223394ec8b0bfd688942a6043 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043 2024-11-22T23:15:30,610 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043, entries=150, sequenceid=367, filesize=12.0 K 2024-11-22T23:15:30,611 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 72b8054778e76f5d48c7bea222a97b27 in 873ms, sequenceid=367, compaction requested=false 2024-11-22T23:15:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:30,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-22T23:15:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-22T23:15:30,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-22T23:15:30,614 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-22T23:15:30,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3350 sec 2024-11-22T23:15:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:30,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:30,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:30,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/89ecfcb95895469ea0f69c2351210a8f is 50, key is test_row_0/A:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:30,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742096_1272 (size=14741) 2024-11-22T23:15:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317390707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317390711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317390710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317390713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317390817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317390828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317390828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:30,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317390828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317391028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317391038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317391039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317391039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/89ecfcb95895469ea0f69c2351210a8f 2024-11-22T23:15:31,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/243a762c39a7409dace5b3318eb82bd1 is 50, key is test_row_0/B:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:31,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742097_1273 (size=12301) 2024-11-22T23:15:31,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/243a762c39a7409dace5b3318eb82bd1 2024-11-22T23:15:31,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cec747caed5f4efc80c7b3875ec5e68f is 50, key is test_row_0/C:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:31,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742098_1274 (size=12301) 2024-11-22T23:15:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317391337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317391347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317391347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317391347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-22T23:15:31,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:31,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-22T23:15:31,429 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:31,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:15:31,430 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:31,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:15:31,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cec747caed5f4efc80c7b3875ec5e68f 2024-11-22T23:15:31,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/89ecfcb95895469ea0f69c2351210a8f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f 2024-11-22T23:15:31,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f, entries=200, sequenceid=383, filesize=14.4 K 2024-11-22T23:15:31,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/243a762c39a7409dace5b3318eb82bd1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1 2024-11-22T23:15:31,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1, entries=150, sequenceid=383, filesize=12.0 K 2024-11-22T23:15:31,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cec747caed5f4efc80c7b3875ec5e68f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f 2024-11-22T23:15:31,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f, entries=150, sequenceid=383, filesize=12.0 K 2024-11-22T23:15:31,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 72b8054778e76f5d48c7bea222a97b27 in 881ms, sequenceid=383, compaction requested=true 2024-11-22T23:15:31,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:31,571 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:31,571 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:31,572 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:31,572 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:31,572 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:31,572 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:31,572 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,572 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,572 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/468f3c2197914208ba0e109adfce1cc5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=38.8 K 2024-11-22T23:15:31,572 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0c969bb0203746acb0723675f27a704b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.4 K 2024-11-22T23:15:31,572 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 468f3c2197914208ba0e109adfce1cc5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:31,574 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb32a2e4e3543459073104203fd2d93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732317328877 2024-11-22T23:15:31,574 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c969bb0203746acb0723675f27a704b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:31,574 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ecfcb95895469ea0f69c2351210a8f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:31,574 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ae941e8f41d4594b50c3eedc5d76e27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732317328877 2024-11-22T23:15:31,574 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 243a762c39a7409dace5b3318eb82bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:31,580 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#236 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:31,580 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/a4d9ae1534df49b48384d6a6a5c0bb15 is 50, key is test_row_0/B:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:31,580 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:31,581 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/82226d2f99c2408eb65a362d00cef38f is 50, key is test_row_0/A:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:31,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-22T23:15:31,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,584 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:31,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742099_1275 (size=12813) 2024-11-22T23:15:31,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/82e9d73f12d8481cb3b59986ce5d5488 is 50, key is test_row_0/A:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:31,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742100_1276 (size=12813) 2024-11-22T23:15:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742101_1277 (size=12301) 2024-11-22T23:15:31,599 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/a4d9ae1534df49b48384d6a6a5c0bb15 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a4d9ae1534df49b48384d6a6a5c0bb15 2024-11-22T23:15:31,601 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/82e9d73f12d8481cb3b59986ce5d5488 2024-11-22T23:15:31,606 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into a4d9ae1534df49b48384d6a6a5c0bb15(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:31,606 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:31,606 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317331571; duration=0sec 2024-11-22T23:15:31,606 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:31,606 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:31,606 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:31,607 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:31,607 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:31,607 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,607 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/be8d9070293a44e29879443d8adbf10a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.4 K 2024-11-22T23:15:31,607 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting be8d9070293a44e29879443d8adbf10a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732317328244 2024-11-22T23:15:31,608 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ec08ea2223394ec8b0bfd688942a6043, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732317328877 2024-11-22T23:15:31,608 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting cec747caed5f4efc80c7b3875ec5e68f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:31,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3d8ac78834674fa18edc988d6fa1ee88 is 50, key is test_row_0/B:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742102_1278 (size=12301) 2024-11-22T23:15:31,614 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3d8ac78834674fa18edc988d6fa1ee88 2024-11-22T23:15:31,617 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:31,618 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ea25ad082a04429dba5d9ce055b59d66 is 50, key is test_row_0/C:col10/1732317330025/Put/seqid=0 2024-11-22T23:15:31,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742103_1279 (size=12813) 2024-11-22T23:15:31,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cbc8b0a8bed9449691d0cd45e4e6b362 is 50, key is test_row_0/C:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:31,628 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ea25ad082a04429dba5d9ce055b59d66 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ea25ad082a04429dba5d9ce055b59d66 2024-11-22T23:15:31,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742104_1280 (size=12301) 2024-11-22T23:15:31,631 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cbc8b0a8bed9449691d0cd45e4e6b362 2024-11-22T23:15:31,635 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into ea25ad082a04429dba5d9ce055b59d66(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:31,635 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:31,635 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317331571; duration=0sec 2024-11-22T23:15:31,635 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:31,635 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/82e9d73f12d8481cb3b59986ce5d5488 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488 2024-11-22T23:15:31,641 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488, entries=150, sequenceid=402, filesize=12.0 K 2024-11-22T23:15:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3d8ac78834674fa18edc988d6fa1ee88 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88 2024-11-22T23:15:31,646 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88, entries=150, sequenceid=402, filesize=12.0 K 2024-11-22T23:15:31,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/cbc8b0a8bed9449691d0cd45e4e6b362 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362 2024-11-22T23:15:31,650 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362, entries=150, sequenceid=402, filesize=12.0 K 2024-11-22T23:15:31,651 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 72b8054778e76f5d48c7bea222a97b27 in 67ms, sequenceid=402, compaction requested=false 2024-11-22T23:15:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-22T23:15:31,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-22T23:15:31,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-22T23:15:31,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 224 msec 2024-11-22T23:15:31,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 230 msec 2024-11-22T23:15:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-22T23:15:31,748 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:31,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-22T23:15:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:31,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:31,750 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:31,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:31,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:31,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:31,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/ad8b1a607fc7407788ec932f5d5b06a9 is 50, key is test_row_1/A:col10/1732317331854/Put/seqid=0 2024-11-22T23:15:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742105_1281 (size=17177) 2024-11-22T23:15:31,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317331873, max timestamp=1732317331882Num uniques 859; 2024-11-22T23:15:31,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:15:31,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317331873, max timestamp=1732317331882Num uniques 859; 2024-11-22T23:15:31,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:31,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317331873, max timestamp=1732317331882Num uniques 859; 2024-11-22T23:15:31,889 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-22T23:15:31,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:31,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:31,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:31,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:31,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317391905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317391910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317391911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317391912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:31,992 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/82226d2f99c2408eb65a362d00cef38f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82226d2f99c2408eb65a362d00cef38f 2024-11-22T23:15:31,997 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 82226d2f99c2408eb65a362d00cef38f(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:31,997 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:31,997 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317331570; duration=0sec 2024-11-22T23:15:31,997 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:31,997 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317392017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317392017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317392028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317392028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:32,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317392228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317392228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317392237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317392237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/ad8b1a607fc7407788ec932f5d5b06a9 2024-11-22T23:15:32,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/499e98caba2943dcbed071798c4c2757 is 50, key is test_row_1/B:col10/1732317331854/Put/seqid=0 2024-11-22T23:15:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742106_1282 (size=9857) 2024-11-22T23:15:32,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/499e98caba2943dcbed071798c4c2757 2024-11-22T23:15:32,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b017ab5e26dc4576b4e92f1573c76ccd is 50, key is test_row_1/C:col10/1732317331854/Put/seqid=0 2024-11-22T23:15:32,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742107_1283 (size=9857) 2024-11-22T23:15:32,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b017ab5e26dc4576b4e92f1573c76ccd 2024-11-22T23:15:32,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/ad8b1a607fc7407788ec932f5d5b06a9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9 2024-11-22T23:15:32,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9, entries=250, sequenceid=415, filesize=16.8 K 2024-11-22T23:15:32,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/499e98caba2943dcbed071798c4c2757 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757 2024-11-22T23:15:32,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757, entries=100, sequenceid=415, filesize=9.6 K 2024-11-22T23:15:32,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/b017ab5e26dc4576b4e92f1573c76ccd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd 2024-11-22T23:15:32,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd, entries=100, sequenceid=415, filesize=9.6 K 2024-11-22T23:15:32,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.31 KB/132415 for 72b8054778e76f5d48c7bea222a97b27 in 508ms, sequenceid=415, compaction requested=true 2024-11-22T23:15:32,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:32,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:32,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:32,370 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,371 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,371 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a4d9ae1534df49b48384d6a6a5c0bb15, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.2 K 2024-11-22T23:15:32,371 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82226d2f99c2408eb65a362d00cef38f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=41.3 K 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d9ae1534df49b48384d6a6a5c0bb15, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82226d2f99c2408eb65a362d00cef38f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d8ac78834674fa18edc988d6fa1ee88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732317330705 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e9d73f12d8481cb3b59986ce5d5488, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732317330705 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad8b1a607fc7407788ec932f5d5b06a9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317331854 2024-11-22T23:15:32,371 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 499e98caba2943dcbed071798c4c2757, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317331854 2024-11-22T23:15:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:32,379 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:32,379 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/b705b6ccc3f241cbbc03beb9dcf8f622 is 50, key is test_row_0/B:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:32,380 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:32,381 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/d084a4fe448a4fe1bf5dfa2e60d3e26b is 50, key is test_row_0/A:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:32,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742108_1284 (size=12915) 2024-11-22T23:15:32,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742109_1285 (size=12915) 2024-11-22T23:15:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:32,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=136.02 KB heapSize=389.15 KB 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:32,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3e25b22ccb6944e2bcb90a6e5b116d5a is 50, key is test_row_0/A:col10/1732317331910/Put/seqid=0 2024-11-22T23:15:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742110_1286 (size=14741) 2024-11-22T23:15:32,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3e25b22ccb6944e2bcb90a6e5b116d5a 2024-11-22T23:15:32,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317392510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/a0158b94e0a949bda3b31840bce3e73a is 50, key is test_row_0/B:col10/1732317331910/Put/seqid=0 2024-11-22T23:15:32,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317392538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317392538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742111_1287 (size=12301) 2024-11-22T23:15:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317392548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317392548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317392618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,790 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/b705b6ccc3f241cbbc03beb9dcf8f622 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/b705b6ccc3f241cbbc03beb9dcf8f622 2024-11-22T23:15:32,795 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/d084a4fe448a4fe1bf5dfa2e60d3e26b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/d084a4fe448a4fe1bf5dfa2e60d3e26b 2024-11-22T23:15:32,795 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into b705b6ccc3f241cbbc03beb9dcf8f622(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:32,795 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:32,795 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317332370; duration=0sec 2024-11-22T23:15:32,795 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:32,796 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:32,796 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:32,797 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:32,797 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:32,797 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,798 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ea25ad082a04429dba5d9ce055b59d66, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.2 K 2024-11-22T23:15:32,798 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ea25ad082a04429dba5d9ce055b59d66, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732317330025 2024-11-22T23:15:32,800 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc8b0a8bed9449691d0cd45e4e6b362, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732317330705 2024-11-22T23:15:32,800 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b017ab5e26dc4576b4e92f1573c76ccd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317331854 2024-11-22T23:15:32,803 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into d084a4fe448a4fe1bf5dfa2e60d3e26b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:32,803 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:32,803 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317332370; duration=0sec 2024-11-22T23:15:32,804 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:32,804 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:32,810 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:32,811 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/80bbabf2d07f41039cadb4c4b346d68b is 50, key is test_row_0/C:col10/1732317330712/Put/seqid=0 2024-11-22T23:15:32,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742112_1288 (size=12915) 2024-11-22T23:15:32,819 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/80bbabf2d07f41039cadb4c4b346d68b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/80bbabf2d07f41039cadb4c4b346d68b 2024-11-22T23:15:32,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,826 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into 80bbabf2d07f41039cadb4c4b346d68b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:32,826 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:32,826 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317332370; duration=0sec 2024-11-22T23:15:32,826 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:32,826 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:32,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317392827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:32,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:32,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/a0158b94e0a949bda3b31840bce3e73a 2024-11-22T23:15:32,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/22b1969d2e98456d83b393167c384d4d is 50, key is test_row_0/C:col10/1732317331910/Put/seqid=0 2024-11-22T23:15:32,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742113_1289 (size=12301) 2024-11-22T23:15:32,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:32,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317393048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317393048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317393068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317393069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317393137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:33,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/22b1969d2e98456d83b393167c384d4d 2024-11-22T23:15:33,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3e25b22ccb6944e2bcb90a6e5b116d5a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a 2024-11-22T23:15:33,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a, entries=200, sequenceid=453, filesize=14.4 K 2024-11-22T23:15:33,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/a0158b94e0a949bda3b31840bce3e73a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a 2024-11-22T23:15:33,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a, entries=150, sequenceid=453, filesize=12.0 K 2024-11-22T23:15:33,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/22b1969d2e98456d83b393167c384d4d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d 2024-11-22T23:15:33,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d, entries=150, sequenceid=453, filesize=12.0 K 2024-11-22T23:15:33,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.02 KB/139285, heapSize ~389.10 KB/398440, currentSize=46.96 KB/48090 for 72b8054778e76f5d48c7bea222a97b27 in 927ms, sequenceid=453, compaction requested=false 2024-11-22T23:15:33,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:33,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-22T23:15:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,439 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:33,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/57ced728031d447f9d146a4d4180c9b1 is 50, key is test_row_0/A:col10/1732317332508/Put/seqid=0 2024-11-22T23:15:33,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742114_1290 (size=12301) 2024-11-22T23:15:33,482 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/57ced728031d447f9d146a4d4180c9b1 2024-11-22T23:15:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3541e9544d3644fbb7f1c5fb03392c6a is 50, key is test_row_0/B:col10/1732317332508/Put/seqid=0 2024-11-22T23:15:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742115_1291 (size=12301) 2024-11-22T23:15:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:33,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317333713, max timestamp=1732317333730Num uniques 859; 2024-11-22T23:15:33,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:33,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317333713, max timestamp=1732317333730Num uniques 859; 2024-11-22T23:15:33,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:33,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317333713, max timestamp=1732317333730Num uniques 859; 2024-11-22T23:15:33,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317393822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:33,897 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3541e9544d3644fbb7f1c5fb03392c6a 2024-11-22T23:15:33,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0d5ed50fee4a440593432ea508eacae3 is 50, key is test_row_0/C:col10/1732317332508/Put/seqid=0 2024-11-22T23:15:33,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742116_1292 (size=12301) 2024-11-22T23:15:33,907 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0d5ed50fee4a440593432ea508eacae3 2024-11-22T23:15:33,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/57ced728031d447f9d146a4d4180c9b1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1 2024-11-22T23:15:33,915 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1, entries=150, sequenceid=466, filesize=12.0 K 2024-11-22T23:15:33,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/3541e9544d3644fbb7f1c5fb03392c6a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a 2024-11-22T23:15:33,919 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a, entries=150, sequenceid=466, filesize=12.0 K 2024-11-22T23:15:33,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/0d5ed50fee4a440593432ea508eacae3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3 2024-11-22T23:15:33,925 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3, entries=150, sequenceid=466, filesize=12.0 K 2024-11-22T23:15:33,926 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 72b8054778e76f5d48c7bea222a97b27 in 487ms, sequenceid=466, compaction requested=true 2024-11-22T23:15:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-22T23:15:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-22T23:15:33,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-22T23:15:33,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-22T23:15:33,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.1830 sec 2024-11-22T23:15:33,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:33,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:33,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/cb095c139db045de9af7c0cec244c1fa is 50, key is test_row_0/A:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:33,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742117_1293 (size=12301) 2024-11-22T23:15:33,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/cb095c139db045de9af7c0cec244c1fa 2024-11-22T23:15:33,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9e019a3a216c4669b2c448f07a199d0a is 50, key is test_row_0/B:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:33,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742118_1294 (size=12301) 2024-11-22T23:15:33,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9e019a3a216c4669b2c448f07a199d0a 2024-11-22T23:15:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317393985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:33,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4a6b6902fdb44937a931c1df739097bc is 50, key is test_row_0/C:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:34,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742119_1295 (size=12301) 2024-11-22T23:15:34,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317394058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317394058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,063 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:34,063 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:34,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317394077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,079 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317394087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,093 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317394098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317394307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=504 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4a6b6902fdb44937a931c1df739097bc 2024-11-22T23:15:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/cb095c139db045de9af7c0cec244c1fa as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa 2024-11-22T23:15:34,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa, entries=150, sequenceid=504, filesize=12.0 K 2024-11-22T23:15:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9e019a3a216c4669b2c448f07a199d0a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a 2024-11-22T23:15:34,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a, entries=150, sequenceid=504, filesize=12.0 K 2024-11-22T23:15:34,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4a6b6902fdb44937a931c1df739097bc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc 2024-11-22T23:15:34,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc, entries=150, sequenceid=504, filesize=12.0 K 2024-11-22T23:15:34,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 72b8054778e76f5d48c7bea222a97b27 in 484ms, sequenceid=504, compaction requested=true 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:34,424 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:15:34,424 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52258 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:34,425 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:34,425 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/d084a4fe448a4fe1bf5dfa2e60d3e26b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=51.0 K 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d084a4fe448a4fe1bf5dfa2e60d3e26b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317330712 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:34,425 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:34,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e25b22ccb6944e2bcb90a6e5b116d5a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732317331903 2024-11-22T23:15:34,425 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/b705b6ccc3f241cbbc03beb9dcf8f622, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=48.7 K 2024-11-22T23:15:34,426 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57ced728031d447f9d146a4d4180c9b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732317332465 2024-11-22T23:15:34,426 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b705b6ccc3f241cbbc03beb9dcf8f622, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317330712 2024-11-22T23:15:34,426 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb095c139db045de9af7c0cec244c1fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:34,426 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a0158b94e0a949bda3b31840bce3e73a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732317331909 2024-11-22T23:15:34,427 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3541e9544d3644fbb7f1c5fb03392c6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732317332465 2024-11-22T23:15:34,427 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e019a3a216c4669b2c448f07a199d0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:34,434 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:34,434 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/2b47d89a6bcd4df2b471308705b65375 is 50, key is test_row_0/B:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:34,437 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:34,438 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/86ac38c997c040ae9a472cc0737ee079 is 50, key is test_row_0/A:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:34,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742120_1296 (size=13051) 2024-11-22T23:15:34,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742121_1297 (size=13051) 2024-11-22T23:15:34,446 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/86ac38c997c040ae9a472cc0737ee079 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/86ac38c997c040ae9a472cc0737ee079 2024-11-22T23:15:34,451 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 86ac38c997c040ae9a472cc0737ee079(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:34,451 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:34,451 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=12, startTime=1732317334424; duration=0sec 2024-11-22T23:15:34,451 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:34,451 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:34,451 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:15:34,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:15:34,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:34,452 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:34,452 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/80bbabf2d07f41039cadb4c4b346d68b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=48.7 K 2024-11-22T23:15:34,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80bbabf2d07f41039cadb4c4b346d68b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732317330712 2024-11-22T23:15:34,452 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b1969d2e98456d83b393167c384d4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1732317331909 2024-11-22T23:15:34,453 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d5ed50fee4a440593432ea508eacae3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732317332465 2024-11-22T23:15:34,453 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a6b6902fdb44937a931c1df739097bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:34,461 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:34,462 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/e336b754614e44fd8c3654007b44e736 is 50, key is test_row_0/C:col10/1732317333820/Put/seqid=0 2024-11-22T23:15:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742122_1298 (size=13051) 2024-11-22T23:15:34,474 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/e336b754614e44fd8c3654007b44e736 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/e336b754614e44fd8c3654007b44e736 2024-11-22T23:15:34,481 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into e336b754614e44fd8c3654007b44e736(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:34,481 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:34,481 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=12, startTime=1732317334424; duration=0sec 2024-11-22T23:15:34,481 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:34,481 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:34,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:34,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:34,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/0ff58a7d58d84a13913baad4451154c9 is 50, key is test_row_0/A:col10/1732317334618/Put/seqid=0 2024-11-22T23:15:34,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742123_1299 (size=14741) 2024-11-22T23:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317334673, max timestamp=1732317334702Num uniques 859; 2024-11-22T23:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317334673, max timestamp=1732317334702Num uniques 859; 2024-11-22T23:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:34,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:34,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317334673, max timestamp=1732317334702Num uniques 859; 2024-11-22T23:15:34,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:34,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317394805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:34,852 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/2b47d89a6bcd4df2b471308705b65375 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2b47d89a6bcd4df2b471308705b65375 2024-11-22T23:15:34,859 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into 2b47d89a6bcd4df2b471308705b65375(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:34,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:34,859 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=12, startTime=1732317334424; duration=0sec 2024-11-22T23:15:34,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:34,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:34,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317394918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:34,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:35,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/0ff58a7d58d84a13913baad4451154c9 2024-11-22T23:15:35,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5dddb9f353ca4bc5b31ceff19ef2a2bd is 50, key is test_row_0/B:col10/1732317334618/Put/seqid=0 2024-11-22T23:15:35,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742124_1300 (size=12301) 2024-11-22T23:15:35,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5dddb9f353ca4bc5b31ceff19ef2a2bd 2024-11-22T23:15:35,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a28df1ba241c4494a542aaa573bb09e5 is 50, key is test_row_0/C:col10/1732317334618/Put/seqid=0 2024-11-22T23:15:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742125_1301 (size=12301) 2024-11-22T23:15:35,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:35,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317395128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:35,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:35,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317395438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:35,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a28df1ba241c4494a542aaa573bb09e5 2024-11-22T23:15:35,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/0ff58a7d58d84a13913baad4451154c9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9 2024-11-22T23:15:35,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9, entries=200, sequenceid=516, filesize=14.4 K 2024-11-22T23:15:35,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5dddb9f353ca4bc5b31ceff19ef2a2bd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd 2024-11-22T23:15:35,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd, entries=150, sequenceid=516, filesize=12.0 K 2024-11-22T23:15:35,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a28df1ba241c4494a542aaa573bb09e5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5 2024-11-22T23:15:35,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5, entries=150, sequenceid=516, filesize=12.0 K 2024-11-22T23:15:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 72b8054778e76f5d48c7bea222a97b27 in 934ms, sequenceid=516, compaction requested=false 2024-11-22T23:15:35,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:35,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-22T23:15:35,918 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:35,919 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-22T23:15:35,922 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:35,923 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:35,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:35,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:35,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:35,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/c5b72531e55545fd86a2133cfecb248a is 50, key is test_row_0/A:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:35,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742126_1302 (size=14741) 2024-11-22T23:15:36,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/c5b72531e55545fd86a2133cfecb248a 2024-11-22T23:15:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317396001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9d04f56f6905405582c8cd0cc38e5856 is 50, key is test_row_0/B:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:36,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742127_1303 (size=12301) 2024-11-22T23:15:36,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T23:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317396078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317396078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,085 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,085 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317396087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,095 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,096 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317396118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317396118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,122 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T23:15:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317396328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:36,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:36,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T23:15:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:36,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9d04f56f6905405582c8cd0cc38e5856 2024-11-22T23:15:36,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7959493c615c4ac4b8167e0f6024c1e1 is 50, key is test_row_0/C:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:36,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742128_1304 (size=12301) 2024-11-22T23:15:36,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7959493c615c4ac4b8167e0f6024c1e1 2024-11-22T23:15:36,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/c5b72531e55545fd86a2133cfecb248a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a 2024-11-22T23:15:36,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a, entries=200, sequenceid=555, filesize=14.4 K 2024-11-22T23:15:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/9d04f56f6905405582c8cd0cc38e5856 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856 2024-11-22T23:15:36,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856, entries=150, sequenceid=555, filesize=12.0 K 2024-11-22T23:15:36,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7959493c615c4ac4b8167e0f6024c1e1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1 2024-11-22T23:15:36,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1, entries=150, sequenceid=555, filesize=12.0 K 2024-11-22T23:15:36,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 72b8054778e76f5d48c7bea222a97b27 in 582ms, sequenceid=555, compaction requested=true 2024-11-22T23:15:36,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:36,536 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:36,536 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:36,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:36,537 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:36,537 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:36,537 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,537 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/86ac38c997c040ae9a472cc0737ee079, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=41.5 K 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:36,538 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,538 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2b47d89a6bcd4df2b471308705b65375, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.8 K 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ac38c997c040ae9a472cc0737ee079, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b47d89a6bcd4df2b471308705b65375, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff58a7d58d84a13913baad4451154c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732317333957 2024-11-22T23:15:36,538 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dddb9f353ca4bc5b31ceff19ef2a2bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732317333965 2024-11-22T23:15:36,539 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5b72531e55545fd86a2133cfecb248a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:36,539 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d04f56f6905405582c8cd0cc38e5856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:36,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-22T23:15:36,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,556 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:36,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:36,557 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:36,557 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/8e97e16ed4f9419bb2c70d72018873cd is 50, key is test_row_0/A:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:36,568 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:36,568 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5a7403ac83b947ad8aad824c9e6b78c8 is 50, key is test_row_0/B:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/28f41fb426a04375b58327162eee5643 is 50, key is test_row_0/A:col10/1732317335999/Put/seqid=0 2024-11-22T23:15:36,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742129_1305 (size=13153) 2024-11-22T23:15:36,626 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/8e97e16ed4f9419bb2c70d72018873cd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/8e97e16ed4f9419bb2c70d72018873cd 2024-11-22T23:15:36,633 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 8e97e16ed4f9419bb2c70d72018873cd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:36,633 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:36,633 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317336536; duration=0sec 2024-11-22T23:15:36,634 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:36,634 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:36,634 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:36,637 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:36,638 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:36,638 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:36,638 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/e336b754614e44fd8c3654007b44e736, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=36.8 K 2024-11-22T23:15:36,638 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting e336b754614e44fd8c3654007b44e736, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=504, earliestPutTs=1732317333814 2024-11-22T23:15:36,639 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a28df1ba241c4494a542aaa573bb09e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=516, earliestPutTs=1732317333965 2024-11-22T23:15:36,639 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7959493c615c4ac4b8167e0f6024c1e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:36,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742130_1306 (size=13153) 2024-11-22T23:15:36,668 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5a7403ac83b947ad8aad824c9e6b78c8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5a7403ac83b947ad8aad824c9e6b78c8 2024-11-22T23:15:36,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742131_1307 (size=9857) 2024-11-22T23:15:36,677 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into 5a7403ac83b947ad8aad824c9e6b78c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:36,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:36,677 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317336536; duration=0sec 2024-11-22T23:15:36,678 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:36,678 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:36,678 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/28f41fb426a04375b58327162eee5643 2024-11-22T23:15:36,686 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:36,686 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/d28a6d1d20bc4479b5c2210a216d2683 is 50, key is test_row_0/C:col10/1732317334791/Put/seqid=0 2024-11-22T23:15:36,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/4c918e9237004449ae1737d4fb277696 is 50, key is test_row_0/B:col10/1732317335999/Put/seqid=0 2024-11-22T23:15:36,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742132_1308 (size=13153) 2024-11-22T23:15:36,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742133_1309 (size=9857) 2024-11-22T23:15:36,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317336719, max timestamp=1732317336767Num uniques 859; 2024-11-22T23:15:36,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:36,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317336719, max timestamp=1732317336767Num uniques 859; 2024-11-22T23:15:36,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317336719, max timestamp=1732317336767Num uniques 859; 2024-11-22T23:15:36,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:36,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:36,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317336844, max timestamp=1732317336875Num uniques 1018; 2024-11-22T23:15:36,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:15:36,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317336844, max timestamp=1732317336875Num uniques 1018; 2024-11-22T23:15:36,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732317336844, max timestamp=1732317336875Num uniques 1018; 2024-11-22T23:15:36,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:15:36,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-22T23:15:36,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732317336948, max timestamp=1732317336966Num uniques 1027; 2024-11-22T23:15:36,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732317336948, max timestamp=1732317336966Num uniques 1027; 2024-11-22T23:15:36,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-22T23:15:36,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-22T23:15:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732317336948, max timestamp=1732317336966Num uniques 1027; 2024-11-22T23:15:36,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-22T23:15:37,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317336987, max timestamp=1732317337049Num uniques 1036; 2024-11-22T23:15:37,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317336987, max timestamp=1732317337049Num uniques 1036; 2024-11-22T23:15:37,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-22T23:15:37,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-22T23:15:37,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732317336987, max timestamp=1732317337049Num uniques 1036; 2024-11-22T23:15:37,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-22T23:15:37,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:37,131 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/4c918e9237004449ae1737d4fb277696 2024-11-22T23:15:37,136 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/d28a6d1d20bc4479b5c2210a216d2683 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/d28a6d1d20bc4479b5c2210a216d2683 2024-11-22T23:15:37,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732317337112, max timestamp=1732317337132Num uniques 1045; 2024-11-22T23:15:37,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-22T23:15:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/19dac17893514c0ab26e58852769c4db is 50, key is test_row_0/C:col10/1732317335999/Put/seqid=0 2024-11-22T23:15:37,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742134_1310 (size=9857) 2024-11-22T23:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317337112, max timestamp=1732317337132Num uniques 1045; 2024-11-22T23:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-22T23:15:37,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732317337112, max timestamp=1732317337132Num uniques 1045; 2024-11-22T23:15:37,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-22T23:15:37,144 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into d28a6d1d20bc4479b5c2210a216d2683(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:37,144 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:37,144 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317336536; duration=0sec 2024-11-22T23:15:37,145 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:37,145 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:37,145 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/19dac17893514c0ab26e58852769c4db 2024-11-22T23:15:37,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/28f41fb426a04375b58327162eee5643 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643 2024-11-22T23:15:37,155 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643, entries=100, sequenceid=564, filesize=9.6 K 2024-11-22T23:15:37,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/4c918e9237004449ae1737d4fb277696 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696 2024-11-22T23:15:37,161 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696, entries=100, sequenceid=564, filesize=9.6 K 2024-11-22T23:15:37,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/19dac17893514c0ab26e58852769c4db as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db 2024-11-22T23:15:37,165 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db, entries=100, sequenceid=564, filesize=9.6 K 2024-11-22T23:15:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-22T23:15:37,168 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=69.78 KB/71450 for 72b8054778e76f5d48c7bea222a97b27 in 612ms, sequenceid=564, compaction requested=false 2024-11-22T23:15:37,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:37,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:37,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-22T23:15:37,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-22T23:15:37,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-22T23:15:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:37,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-22T23:15:37,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=76.48 KB heapSize=213.09 KB 2024-11-22T23:15:37,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:37,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:15:37,174 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.2540 sec 2024-11-22T23:15:37,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2af5636bd73148bb94e57407ef92ca49 is 50, key is test_row_0/A:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:37,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742135_1311 (size=12301) 2024-11-22T23:15:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 307 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317397261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:37,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317397377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:37,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2af5636bd73148bb94e57407ef92ca49 2024-11-22T23:15:37,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f7cf8d012e244477a63cac5c0dde2147 is 50, key is test_row_0/B:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:37,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:37,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317397590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:37,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742136_1312 (size=12301) 2024-11-22T23:15:37,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317397907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:38,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.82 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f7cf8d012e244477a63cac5c0dde2147 2024-11-22T23:15:38,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/446ec4f49db84e1d81435ea97a24ea55 is 50, key is test_row_0/C:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:38,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742137_1313 (size=12301) 2024-11-22T23:15:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-22T23:15:38,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:38,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-22T23:15:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T23:15:38,070 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:38,071 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:38,071 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T23:15:38,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-22T23:15:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-22T23:15:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T23:15:38,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.82 KB at sequenceid=664 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/446ec4f49db84e1d81435ea97a24ea55 2024-11-22T23:15:38,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317398417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:38,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:38,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2af5636bd73148bb94e57407ef92ca49 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49 2024-11-22T23:15:38,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49, entries=150, sequenceid=664, filesize=12.0 K 2024-11-22T23:15:38,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/f7cf8d012e244477a63cac5c0dde2147 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147 2024-11-22T23:15:38,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147, entries=150, sequenceid=664, filesize=12.0 K 2024-11-22T23:15:38,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/446ec4f49db84e1d81435ea97a24ea55 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55 2024-11-22T23:15:38,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55, entries=150, sequenceid=664, filesize=12.0 K 2024-11-22T23:15:38,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~76.48 KB/78320, heapSize ~213.05 KB/218160, currentSize=100.63 KB/103050 for 72b8054778e76f5d48c7bea222a97b27 in 1264ms, sequenceid=664, compaction requested=true 2024-11-22T23:15:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:38,435 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:38,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:38,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:38,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:15:38,436 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:38,441 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:38,441 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:38,441 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,441 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/8e97e16ed4f9419bb2c70d72018873cd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.5 K 2024-11-22T23:15:38,441 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:38,441 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:38,441 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,442 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5a7403ac83b947ad8aad824c9e6b78c8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.5 K 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a7403ac83b947ad8aad824c9e6b78c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e97e16ed4f9419bb2c70d72018873cd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c918e9237004449ae1737d4fb277696, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732317335984 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28f41fb426a04375b58327162eee5643, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732317335984 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2af5636bd73148bb94e57407ef92ca49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:38,442 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f7cf8d012e244477a63cac5c0dde2147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:38,449 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:38,450 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:38,450 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/ba798e4c2044446882360d8f15ff3874 is 50, key is test_row_0/B:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:38,450 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/6608583fa974415990717c73bb4b83b7 is 50, key is test_row_0/A:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:38,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742138_1314 (size=13255) 2024-11-22T23:15:38,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742139_1315 (size=13255) 2024-11-22T23:15:38,474 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/ba798e4c2044446882360d8f15ff3874 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ba798e4c2044446882360d8f15ff3874 2024-11-22T23:15:38,478 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into ba798e4c2044446882360d8f15ff3874(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:38,478 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:38,478 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317338435; duration=0sec 2024-11-22T23:15:38,479 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:38,479 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:38,479 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:38,479 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:38,480 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:38,480 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,480 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/d28a6d1d20bc4479b5c2210a216d2683, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.5 K 2024-11-22T23:15:38,480 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d28a6d1d20bc4479b5c2210a216d2683, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732317334760 2024-11-22T23:15:38,480 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 19dac17893514c0ab26e58852769c4db, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1732317335984 2024-11-22T23:15:38,480 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 446ec4f49db84e1d81435ea97a24ea55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:38,485 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:38,486 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a2e9b3ad81f548a58a03d24eebd6c28c is 50, key is test_row_0/C:col10/1732317337170/Put/seqid=0 2024-11-22T23:15:38,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742140_1316 (size=13255) 2024-11-22T23:15:38,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,529 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:38,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/44a3d77df8de40eea1cb30078a5d0076 is 50, key is test_row_0/A:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:38,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742141_1317 (size=12301) 2024-11-22T23:15:38,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/44a3d77df8de40eea1cb30078a5d0076 2024-11-22T23:15:38,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/656bf843967041abaece9b1a973a286e is 50, key is test_row_0/B:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742142_1318 (size=12301) 2024-11-22T23:15:38,554 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/656bf843967041abaece9b1a973a286e 2024-11-22T23:15:38,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7eb6acf99e374969a3f60cefa6c4b765 is 50, key is test_row_0/C:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:38,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742143_1319 (size=12301) 2024-11-22T23:15:38,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T23:15:38,870 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/6608583fa974415990717c73bb4b83b7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6608583fa974415990717c73bb4b83b7 2024-11-22T23:15:38,874 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 6608583fa974415990717c73bb4b83b7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:38,874 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:38,874 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317338435; duration=0sec 2024-11-22T23:15:38,874 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:38,874 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:38,893 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/a2e9b3ad81f548a58a03d24eebd6c28c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2e9b3ad81f548a58a03d24eebd6c28c 2024-11-22T23:15:38,897 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into a2e9b3ad81f548a58a03d24eebd6c28c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:38,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:38,897 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317338436; duration=0sec 2024-11-22T23:15:38,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:38,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:38,970 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7eb6acf99e374969a3f60cefa6c4b765 2024-11-22T23:15:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/44a3d77df8de40eea1cb30078a5d0076 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076 2024-11-22T23:15:38,977 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076, entries=150, sequenceid=683, filesize=12.0 K 2024-11-22T23:15:38,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/656bf843967041abaece9b1a973a286e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e 2024-11-22T23:15:38,981 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e, entries=150, sequenceid=683, filesize=12.0 K 2024-11-22T23:15:38,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/7eb6acf99e374969a3f60cefa6c4b765 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765 2024-11-22T23:15:38,985 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765, entries=150, sequenceid=683, filesize=12.0 K 2024-11-22T23:15:38,986 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 72b8054778e76f5d48c7bea222a97b27 in 457ms, sequenceid=683, compaction requested=false 2024-11-22T23:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:38,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-22T23:15:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-22T23:15:38,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-22T23:15:38,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 916 msec 2024-11-22T23:15:38,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 922 msec 2024-11-22T23:15:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-22T23:15:39,208 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:39,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-22T23:15:39,210 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T23:15:39,211 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:39,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T23:15:39,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-22T23:15:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-22T23:15:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-22T23:15:39,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-22T23:15:39,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-22T23:15:39,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 161 msec 2024-11-22T23:15:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:39,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:39,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:39,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/33ec4b485ec04ed2a46431eeeafccf08 is 50, key is test_row_1/A:col10/1732317339450/Put/seqid=0 2024-11-22T23:15:39,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742144_1320 (size=14737) 2024-11-22T23:15:39,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=696 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/33ec4b485ec04ed2a46431eeeafccf08 2024-11-22T23:15:39,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/94f76f7bf5d04fe7b2fd6966c9c3774c is 50, key is test_row_1/B:col10/1732317339450/Put/seqid=0 2024-11-22T23:15:39,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742145_1321 (size=9857) 2024-11-22T23:15:39,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=696 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/94f76f7bf5d04fe7b2fd6966c9c3774c 2024-11-22T23:15:39,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ff2b99623f164a1d8709b2e094140079 is 50, key is test_row_1/C:col10/1732317339450/Put/seqid=0 2024-11-22T23:15:39,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742146_1322 (size=9857) 2024-11-22T23:15:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-22T23:15:39,527 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:39,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees 2024-11-22T23:15:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:39,529 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:39,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=94, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:39,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:39,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317339492, max timestamp=1732317339531Num uniques 859; 2024-11-22T23:15:39,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317339492, max timestamp=1732317339531Num uniques 859; 2024-11-22T23:15:39,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:39,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:39,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317339492, max timestamp=1732317339531Num uniques 859; 2024-11-22T23:15:39,536 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:15:39,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317399610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:39,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317399718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:39,807 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:15:39,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:39,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317399927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:39,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:39,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:39,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=696 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ff2b99623f164a1d8709b2e094140079 2024-11-22T23:15:40,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/33ec4b485ec04ed2a46431eeeafccf08 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08 2024-11-22T23:15:40,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08, entries=200, sequenceid=696, filesize=14.4 K 2024-11-22T23:15:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/94f76f7bf5d04fe7b2fd6966c9c3774c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c 2024-11-22T23:15:40,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c, entries=100, sequenceid=696, filesize=9.6 K 2024-11-22T23:15:40,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ff2b99623f164a1d8709b2e094140079 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079 2024-11-22T23:15:40,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079, entries=100, sequenceid=696, filesize=9.6 K 2024-11-22T23:15:40,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 72b8054778e76f5d48c7bea222a97b27 in 570ms, sequenceid=696, compaction requested=true 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:15:40,022 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:15:40,022 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 72b8054778e76f5d48c7bea222a97b27:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:15:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/A is initiating minor compaction (all files) 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/B is initiating minor compaction (all files) 2024-11-22T23:15:40,023 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/B in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,023 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/A in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,023 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ba798e4c2044446882360d8f15ff3874, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.6 K 2024-11-22T23:15:40,023 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6608583fa974415990717c73bb4b83b7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=39.3 K 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ba798e4c2044446882360d8f15ff3874, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:40,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6608583fa974415990717c73bb4b83b7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:40,024 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 656bf843967041abaece9b1a973a286e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1732317337229 2024-11-22T23:15:40,024 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44a3d77df8de40eea1cb30078a5d0076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1732317337229 2024-11-22T23:15:40,024 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 94f76f7bf5d04fe7b2fd6966c9c3774c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=696, earliestPutTs=1732317339450 2024-11-22T23:15:40,024 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33ec4b485ec04ed2a46431eeeafccf08, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=696, earliestPutTs=1732317339443 2024-11-22T23:15:40,030 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:40,030 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:40,031 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5ec236126c9a498d83da73a0a6dfc091 is 50, key is test_row_0/B:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:40,031 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/99b3ec7c1f364ece9686e07c8a76f7de is 50, key is test_row_0/A:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742148_1324 (size=13357) 2024-11-22T23:15:40,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742147_1323 (size=13357) 2024-11-22T23:15:40,039 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/5ec236126c9a498d83da73a0a6dfc091 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ec236126c9a498d83da73a0a6dfc091 2024-11-22T23:15:40,052 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/B of 72b8054778e76f5d48c7bea222a97b27 into 5ec236126c9a498d83da73a0a6dfc091(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:40,052 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:40,052 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/B, priority=13, startTime=1732317340022; duration=0sec 2024-11-22T23:15:40,053 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:15:40,053 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:B 2024-11-22T23:15:40,053 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:15:40,054 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:15:40,054 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 72b8054778e76f5d48c7bea222a97b27/C is initiating minor compaction (all files) 2024-11-22T23:15:40,054 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 72b8054778e76f5d48c7bea222a97b27/C in TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,054 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2e9b3ad81f548a58a03d24eebd6c28c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp, totalSize=34.6 K 2024-11-22T23:15:40,054 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e9b3ad81f548a58a03d24eebd6c28c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=664, earliestPutTs=1732317337152 2024-11-22T23:15:40,055 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eb6acf99e374969a3f60cefa6c4b765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1732317337229 2024-11-22T23:15:40,055 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ff2b99623f164a1d8709b2e094140079, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=696, earliestPutTs=1732317339450 2024-11-22T23:15:40,060 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 72b8054778e76f5d48c7bea222a97b27#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:15:40,061 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/58f294b71310438d9f669fa9c3a0862b is 50, key is test_row_0/C:col10/1732317337253/Put/seqid=0 2024-11-22T23:15:40,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742149_1325 (size=13357) 2024-11-22T23:15:40,068 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/58f294b71310438d9f669fa9c3a0862b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/58f294b71310438d9f669fa9c3a0862b 2024-11-22T23:15:40,073 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/C of 72b8054778e76f5d48c7bea222a97b27 into 58f294b71310438d9f669fa9c3a0862b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:40,073 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:40,073 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/C, priority=13, startTime=1732317340022; duration=0sec 2024-11-22T23:15:40,073 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:40,073 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:C 2024-11-22T23:15:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:40,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:15:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:40,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:40,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:15:40,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2c1b646a807045b98d6fe1ee8cbd84a3 is 50, key is test_row_0/A:col10/1732317339599/Put/seqid=0 2024-11-22T23:15:40,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742150_1326 (size=14741) 2024-11-22T23:15:40,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317400121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317400126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34908 deadline: 1732317400137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,142 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34902 deadline: 1732317400147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,151 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 72b8054778e76f5d48c7bea222a97b27 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:40,235 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-22T23:15:40,235 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:40,235 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,235 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:40,235 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-22T23:15:40,236 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:40,236 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,236 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317400237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,238 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-22T23:15:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,238 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,238 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317400238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317400238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,239 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,240 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-22T23:15:40,240 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:40,240 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,240 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:40,241 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-22T23:15:40,241 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:40,241 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,241 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:40,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:40,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,440 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/99b3ec7c1f364ece9686e07c8a76f7de as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/99b3ec7c1f364ece9686e07c8a76f7de 2024-11-22T23:15:40,445 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 72b8054778e76f5d48c7bea222a97b27/A of 72b8054778e76f5d48c7bea222a97b27 into 99b3ec7c1f364ece9686e07c8a76f7de(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:15:40,445 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:40,445 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., storeName=72b8054778e76f5d48c7bea222a97b27/A, priority=13, startTime=1732317340022; duration=0sec 2024-11-22T23:15:40,446 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:15:40,446 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 72b8054778e76f5d48c7bea222a97b27:A 2024-11-22T23:15:40,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317400448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317400448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2c1b646a807045b98d6fe1ee8cbd84a3 2024-11-22T23:15:40,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/16bfffe3f24d4dafa61e0e4dfa6987ea is 50, key is test_row_0/B:col10/1732317339599/Put/seqid=0 2024-11-22T23:15:40,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742151_1327 (size=12301) 2024-11-22T23:15:40,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:40,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 357 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34912 deadline: 1732317400747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317400758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317400758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:40,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:40,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:40,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:40,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:40,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/16bfffe3f24d4dafa61e0e4dfa6987ea 2024-11-22T23:15:40,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ae362501ccb24362af970e3f8c3961c3 is 50, key is test_row_0/C:col10/1732317339599/Put/seqid=0 2024-11-22T23:15:40,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742152_1328 (size=12301) 2024-11-22T23:15:41,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:41,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:41,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:41,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:41,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:41,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] handler.RSProcedureHandler(58): pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=95 java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=95 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34876 deadline: 1732317401268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34842 deadline: 1732317401268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:15:41,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=736 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ae362501ccb24362af970e3f8c3961c3 2024-11-22T23:15:41,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/2c1b646a807045b98d6fe1ee8cbd84a3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2c1b646a807045b98d6fe1ee8cbd84a3 2024-11-22T23:15:41,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2c1b646a807045b98d6fe1ee8cbd84a3, entries=200, sequenceid=736, filesize=14.4 K 2024-11-22T23:15:41,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/16bfffe3f24d4dafa61e0e4dfa6987ea as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/16bfffe3f24d4dafa61e0e4dfa6987ea 2024-11-22T23:15:41,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/16bfffe3f24d4dafa61e0e4dfa6987ea, entries=150, sequenceid=736, filesize=12.0 K 2024-11-22T23:15:41,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/ae362501ccb24362af970e3f8c3961c3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ae362501ccb24362af970e3f8c3961c3 2024-11-22T23:15:41,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ae362501ccb24362af970e3f8c3961c3, entries=150, sequenceid=736, filesize=12.0 K 2024-11-22T23:15:41,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 72b8054778e76f5d48c7bea222a97b27 in 1258ms, sequenceid=736, compaction requested=false 2024-11-22T23:15:41,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:41,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=95 2024-11-22T23:15:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:41,371 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:41,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3ca26682923640d3a593490dc1c7fc4f is 50, key is test_row_0/A:col10/1732317340120/Put/seqid=0 2024-11-22T23:15:41,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742153_1329 (size=12301) 2024-11-22T23:15:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:41,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. as already flushing 2024-11-22T23:15:41,763 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-22T23:15:41,763 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:41,763 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:41,763 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:41,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:41,785 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3ca26682923640d3a593490dc1c7fc4f 2024-11-22T23:15:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0f823892db114e69bbef3d8e28b68f10 is 50, key is test_row_0/B:col10/1732317340120/Put/seqid=0 2024-11-22T23:15:41,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742154_1330 (size=12301) 2024-11-22T23:15:42,202 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0f823892db114e69bbef3d8e28b68f10 2024-11-22T23:15:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/62b3254d5371419cb5a0eea2cb386388 is 50, key is test_row_0/C:col10/1732317340120/Put/seqid=0 2024-11-22T23:15:42,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742155_1331 (size=12301) 2024-11-22T23:15:42,292 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-22T23:15:42,292 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-22T23:15:42,292 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:42,292 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:42,292 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:42,292 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:42,292 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:42,292 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:42,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:42,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:42,617 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=746 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/62b3254d5371419cb5a0eea2cb386388 2024-11-22T23:15:42,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/3ca26682923640d3a593490dc1c7fc4f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3ca26682923640d3a593490dc1c7fc4f 2024-11-22T23:15:42,629 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3ca26682923640d3a593490dc1c7fc4f, entries=150, sequenceid=746, filesize=12.0 K 2024-11-22T23:15:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/0f823892db114e69bbef3d8e28b68f10 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0f823892db114e69bbef3d8e28b68f10 2024-11-22T23:15:42,633 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0f823892db114e69bbef3d8e28b68f10, entries=150, sequenceid=746, filesize=12.0 K 2024-11-22T23:15:42,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/62b3254d5371419cb5a0eea2cb386388 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/62b3254d5371419cb5a0eea2cb386388 2024-11-22T23:15:42,637 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/62b3254d5371419cb5a0eea2cb386388, entries=150, sequenceid=746, filesize=12.0 K 2024-11-22T23:15:42,637 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=20.13 KB/20610 for 72b8054778e76f5d48c7bea222a97b27 in 1266ms, sequenceid=746, compaction requested=true 2024-11-22T23:15:42,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.HRegion(2603): Flush status journal for 72b8054778e76f5d48c7bea222a97b27: 2024-11-22T23:15:42,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:42,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=95}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=95 2024-11-22T23:15:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=95 2024-11-22T23:15:42,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-22T23:15:42,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1090 sec 2024-11-22T23:15:42,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=94, table=TestAcidGuarantees in 3.1130 sec 2024-11-22T23:15:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-22T23:15:43,698 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:50,200 INFO [Thread-1075 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1075. 2024-11-22T23:15:50,200 DEBUG [Thread-1075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:50,200 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,200 DEBUG [Thread-1075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,200 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:50,239 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-22T23:15:50,239 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:15:50,239 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,239 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 307 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2848 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2796 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2802 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2835 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2792 2024-11-22T23:15:50,239 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:15:50,239 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:15:50,239 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:15:50,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:50,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:15:50,240 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:15:50,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:50,244 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317350243"}]},"ts":"1732317350243"} 2024-11-22T23:15:50,245 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:15:50,245 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:15:50,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:15:50,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, UNASSIGN}] 2024-11-22T23:15:50,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, UNASSIGN 2024-11-22T23:15:50,250 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=72b8054778e76f5d48c7bea222a97b27, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:50,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, UNASSIGN because future has completed 2024-11-22T23:15:50,252 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:15:50,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:50,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:50,405 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:50,405 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:15:50,405 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 72b8054778e76f5d48c7bea222a97b27, disabling compactions & flushes 2024-11-22T23:15:50,405 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:50,405 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:50,405 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. after waiting 0 ms 2024-11-22T23:15:50,405 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:50,406 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(2902): Flushing 72b8054778e76f5d48c7bea222a97b27 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=A 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=B 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 72b8054778e76f5d48c7bea222a97b27, store=C 2024-11-22T23:15:50,406 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:50,409 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/788e7e707c6543e882539e4400573adb is 50, key is test_row_0/A:col10/1732317342289/Put/seqid=0 2024-11-22T23:15:50,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742156_1332 (size=12301) 2024-11-22T23:15:50,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:50,813 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/788e7e707c6543e882539e4400573adb 2024-11-22T23:15:50,819 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/39ae39af8456434bbe32772b78a0389e is 50, key is test_row_0/B:col10/1732317342289/Put/seqid=0 2024-11-22T23:15:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742157_1333 (size=12301) 2024-11-22T23:15:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:51,224 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/39ae39af8456434bbe32772b78a0389e 2024-11-22T23:15:51,230 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4330eff1e2174853a07397adbb14da60 is 50, key is test_row_0/C:col10/1732317342289/Put/seqid=0 2024-11-22T23:15:51,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742158_1334 (size=12301) 2024-11-22T23:15:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:51,634 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4330eff1e2174853a07397adbb14da60 2024-11-22T23:15:51,641 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/A/788e7e707c6543e882539e4400573adb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/788e7e707c6543e882539e4400573adb 2024-11-22T23:15:51,646 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/788e7e707c6543e882539e4400573adb, entries=150, sequenceid=754, filesize=12.0 K 2024-11-22T23:15:51,647 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/B/39ae39af8456434bbe32772b78a0389e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/39ae39af8456434bbe32772b78a0389e 2024-11-22T23:15:51,651 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/39ae39af8456434bbe32772b78a0389e, entries=150, sequenceid=754, filesize=12.0 K 2024-11-22T23:15:51,652 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/.tmp/C/4330eff1e2174853a07397adbb14da60 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4330eff1e2174853a07397adbb14da60 2024-11-22T23:15:51,657 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4330eff1e2174853a07397adbb14da60, entries=150, sequenceid=754, filesize=12.0 K 2024-11-22T23:15:51,657 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 72b8054778e76f5d48c7bea222a97b27 in 1252ms, sequenceid=754, compaction requested=true 2024-11-22T23:15:51,658 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/f86e709676e24d77b13f7f035dab52f2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44135002df294bd2bfc14caf0783b9d5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/468f3c2197914208ba0e109adfce1cc5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82226d2f99c2408eb65a362d00cef38f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/d084a4fe448a4fe1bf5dfa2e60d3e26b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/86ac38c997c040ae9a472cc0737ee079, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/8e97e16ed4f9419bb2c70d72018873cd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6608583fa974415990717c73bb4b83b7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08] to archive 2024-11-22T23:15:51,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:51,661 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585812d9220844bbb568589114ea7c53 2024-11-22T23:15:51,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/93a46d3c4bd249579990cfdf0e1773c5 2024-11-22T23:15:51,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4ac4a7cab61946f3ae1d4504eb5bee1c 2024-11-22T23:15:51,667 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7c4babe2369245f5a40e071a254c959d 2024-11-22T23:15:51,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/732c0b308da64ab89430be1569b53111 2024-11-22T23:15:51,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/585fae0c13b74a2bb25fd50d0783fec3 2024-11-22T23:15:51,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/f86e709676e24d77b13f7f035dab52f2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/f86e709676e24d77b13f7f035dab52f2 2024-11-22T23:15:51,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4c767a5650394dff8dd6be441718931d 2024-11-22T23:15:51,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/fb1f4e9912854ee2a01ef7147eaf2af2 2024-11-22T23:15:51,673 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44135002df294bd2bfc14caf0783b9d5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44135002df294bd2bfc14caf0783b9d5 2024-11-22T23:15:51,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/7655bb68b87c47768e03a7e035112a16 2024-11-22T23:15:51,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/4d0a7f63df7740cb9d9437350d9184f5 2024-11-22T23:15:51,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/468f3c2197914208ba0e109adfce1cc5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/468f3c2197914208ba0e109adfce1cc5 2024-11-22T23:15:51,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6bb32a2e4e3543459073104203fd2d93 2024-11-22T23:15:51,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/89ecfcb95895469ea0f69c2351210a8f 2024-11-22T23:15:51,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82226d2f99c2408eb65a362d00cef38f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82226d2f99c2408eb65a362d00cef38f 2024-11-22T23:15:51,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/82e9d73f12d8481cb3b59986ce5d5488 2024-11-22T23:15:51,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/ad8b1a607fc7407788ec932f5d5b06a9 2024-11-22T23:15:51,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/d084a4fe448a4fe1bf5dfa2e60d3e26b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/d084a4fe448a4fe1bf5dfa2e60d3e26b 2024-11-22T23:15:51,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3e25b22ccb6944e2bcb90a6e5b116d5a 2024-11-22T23:15:51,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/57ced728031d447f9d146a4d4180c9b1 2024-11-22T23:15:51,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/86ac38c997c040ae9a472cc0737ee079 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/86ac38c997c040ae9a472cc0737ee079 2024-11-22T23:15:51,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/cb095c139db045de9af7c0cec244c1fa 2024-11-22T23:15:51,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/0ff58a7d58d84a13913baad4451154c9 2024-11-22T23:15:51,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/c5b72531e55545fd86a2133cfecb248a 2024-11-22T23:15:51,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/8e97e16ed4f9419bb2c70d72018873cd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/8e97e16ed4f9419bb2c70d72018873cd 2024-11-22T23:15:51,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/28f41fb426a04375b58327162eee5643 2024-11-22T23:15:51,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6608583fa974415990717c73bb4b83b7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/6608583fa974415990717c73bb4b83b7 2024-11-22T23:15:51,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2af5636bd73148bb94e57407ef92ca49 2024-11-22T23:15:51,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/44a3d77df8de40eea1cb30078a5d0076 2024-11-22T23:15:51,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/33ec4b485ec04ed2a46431eeeafccf08 2024-11-22T23:15:51,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/dbf3465fe9184f6f9a1e30e5a64fe4c6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6df0428ecb8d4df1b011e217e2212bf3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0c969bb0203746acb0723675f27a704b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a4d9ae1534df49b48384d6a6a5c0bb15, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/b705b6ccc3f241cbbc03beb9dcf8f622, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2b47d89a6bcd4df2b471308705b65375, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5a7403ac83b947ad8aad824c9e6b78c8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ba798e4c2044446882360d8f15ff3874, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c] to archive 2024-11-22T23:15:51,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:51,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0ac64ea7e6cd47a7a54d18ea03da2371 2024-11-22T23:15:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6b6d20b7ab004679b4dfc53493baab1e 2024-11-22T23:15:51,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2997517ddf25489fad9126e7f230a8c6 2024-11-22T23:15:51,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/aeebbe6a25024f8fa6d16e9f269f4010 2024-11-22T23:15:51,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9f96b456f58c4868b3f7e3e6e506216d 2024-11-22T23:15:51,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/dbf3465fe9184f6f9a1e30e5a64fe4c6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/dbf3465fe9184f6f9a1e30e5a64fe4c6 2024-11-22T23:15:51,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/fadd83627198459b8823d467556621dd 2024-11-22T23:15:51,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f0d8e1d1ab244a64879575425775f207 2024-11-22T23:15:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6df0428ecb8d4df1b011e217e2212bf3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/6df0428ecb8d4df1b011e217e2212bf3 2024-11-22T23:15:51,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ee1f7fbaed9142d3b83a26a156cfa22c 2024-11-22T23:15:51,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/da6bca73bd6d48ef83f29384b577151d 2024-11-22T23:15:51,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0c969bb0203746acb0723675f27a704b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0c969bb0203746acb0723675f27a704b 2024-11-22T23:15:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5d325dc04f174d05b8657152d5b91f3e 2024-11-22T23:15:51,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ae941e8f41d4594b50c3eedc5d76e27 2024-11-22T23:15:51,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a4d9ae1534df49b48384d6a6a5c0bb15 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a4d9ae1534df49b48384d6a6a5c0bb15 2024-11-22T23:15:51,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/243a762c39a7409dace5b3318eb82bd1 2024-11-22T23:15:51,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3d8ac78834674fa18edc988d6fa1ee88 2024-11-22T23:15:51,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/b705b6ccc3f241cbbc03beb9dcf8f622 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/b705b6ccc3f241cbbc03beb9dcf8f622 2024-11-22T23:15:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/499e98caba2943dcbed071798c4c2757 2024-11-22T23:15:51,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/a0158b94e0a949bda3b31840bce3e73a 2024-11-22T23:15:51,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/3541e9544d3644fbb7f1c5fb03392c6a 2024-11-22T23:15:51,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2b47d89a6bcd4df2b471308705b65375 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/2b47d89a6bcd4df2b471308705b65375 2024-11-22T23:15:51,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9e019a3a216c4669b2c448f07a199d0a 2024-11-22T23:15:51,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5dddb9f353ca4bc5b31ceff19ef2a2bd 2024-11-22T23:15:51,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5a7403ac83b947ad8aad824c9e6b78c8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5a7403ac83b947ad8aad824c9e6b78c8 2024-11-22T23:15:51,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/9d04f56f6905405582c8cd0cc38e5856 2024-11-22T23:15:51,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/4c918e9237004449ae1737d4fb277696 2024-11-22T23:15:51,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ba798e4c2044446882360d8f15ff3874 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/ba798e4c2044446882360d8f15ff3874 2024-11-22T23:15:51,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/f7cf8d012e244477a63cac5c0dde2147 2024-11-22T23:15:51,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/656bf843967041abaece9b1a973a286e 2024-11-22T23:15:51,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/94f76f7bf5d04fe7b2fd6966c9c3774c 2024-11-22T23:15:51,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/1196fca6bcca446086787d018e51908f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/9f68aae388914b01957f43cdecd00583, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/be8d9070293a44e29879443d8adbf10a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ea25ad082a04429dba5d9ce055b59d66, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/80bbabf2d07f41039cadb4c4b346d68b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/e336b754614e44fd8c3654007b44e736, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/d28a6d1d20bc4479b5c2210a216d2683, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2e9b3ad81f548a58a03d24eebd6c28c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079] to archive 2024-11-22T23:15:51,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:15:51,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a62da71308af49248f315bac8b781b80 2024-11-22T23:15:51,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2982c7f3a3c4aefb0f5eac445bd453e 2024-11-22T23:15:51,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/88b6683b04b24a2987f55a40e71ce0d6 2024-11-22T23:15:51,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b96998453709460f821a902b55b362ce 2024-11-22T23:15:51,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cf4059aa6c734119b9b96b32edf56c73 2024-11-22T23:15:51,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/1196fca6bcca446086787d018e51908f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/1196fca6bcca446086787d018e51908f 2024-11-22T23:15:51,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/514d96c9daf34debac617d5f3f8b054b 2024-11-22T23:15:51,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/6c8b0fe93d0d4021aaf5fa3590f654ac 2024-11-22T23:15:51,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/9f68aae388914b01957f43cdecd00583 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/9f68aae388914b01957f43cdecd00583 2024-11-22T23:15:51,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/71a8865ed24244b9994da5db0e25542a 2024-11-22T23:15:51,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0b62fd5d3f5647feb98a1d79fcb58349 2024-11-22T23:15:51,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/be8d9070293a44e29879443d8adbf10a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/be8d9070293a44e29879443d8adbf10a 2024-11-22T23:15:51,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/8bd4f522ad6c4683a1761a8ed33c44e1 2024-11-22T23:15:51,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ec08ea2223394ec8b0bfd688942a6043 2024-11-22T23:15:51,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ea25ad082a04429dba5d9ce055b59d66 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ea25ad082a04429dba5d9ce055b59d66 2024-11-22T23:15:51,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cec747caed5f4efc80c7b3875ec5e68f 2024-11-22T23:15:51,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/cbc8b0a8bed9449691d0cd45e4e6b362 2024-11-22T23:15:51,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/80bbabf2d07f41039cadb4c4b346d68b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/80bbabf2d07f41039cadb4c4b346d68b 2024-11-22T23:15:51,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/b017ab5e26dc4576b4e92f1573c76ccd 2024-11-22T23:15:51,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/22b1969d2e98456d83b393167c384d4d 2024-11-22T23:15:51,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/0d5ed50fee4a440593432ea508eacae3 2024-11-22T23:15:51,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/e336b754614e44fd8c3654007b44e736 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/e336b754614e44fd8c3654007b44e736 2024-11-22T23:15:51,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4a6b6902fdb44937a931c1df739097bc 2024-11-22T23:15:51,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a28df1ba241c4494a542aaa573bb09e5 2024-11-22T23:15:51,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/d28a6d1d20bc4479b5c2210a216d2683 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/d28a6d1d20bc4479b5c2210a216d2683 2024-11-22T23:15:51,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7959493c615c4ac4b8167e0f6024c1e1 2024-11-22T23:15:51,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/19dac17893514c0ab26e58852769c4db 2024-11-22T23:15:51,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2e9b3ad81f548a58a03d24eebd6c28c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/a2e9b3ad81f548a58a03d24eebd6c28c 2024-11-22T23:15:51,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/446ec4f49db84e1d81435ea97a24ea55 2024-11-22T23:15:51,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/7eb6acf99e374969a3f60cefa6c4b765 2024-11-22T23:15:51,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ff2b99623f164a1d8709b2e094140079 2024-11-22T23:15:51,778 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/recovered.edits/757.seqid, newMaxSeqId=757, maxSeqId=1 2024-11-22T23:15:51,779 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. 2024-11-22T23:15:51,779 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 72b8054778e76f5d48c7bea222a97b27: Waiting for close lock at 1732317350405Running coprocessor pre-close hooks at 1732317350405Disabling compacts and flushes for region at 1732317350405Disabling writes for close at 1732317350405Obtaining lock to block concurrent updates at 1732317350406 (+1 ms)Preparing flush snapshotting stores in 72b8054778e76f5d48c7bea222a97b27 at 1732317350406Finished memstore snapshotting TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732317350406Flushing stores of TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27. at 1732317350406Flushing 72b8054778e76f5d48c7bea222a97b27/A: creating writer at 1732317350406Flushing 72b8054778e76f5d48c7bea222a97b27/A: appending metadata at 1732317350409 (+3 ms)Flushing 72b8054778e76f5d48c7bea222a97b27/A: closing flushed file at 1732317350409Flushing 72b8054778e76f5d48c7bea222a97b27/B: creating writer at 1732317350817 (+408 ms)Flushing 72b8054778e76f5d48c7bea222a97b27/B: appending metadata at 1732317350819 (+2 ms)Flushing 72b8054778e76f5d48c7bea222a97b27/B: closing flushed file at 1732317350819Flushing 72b8054778e76f5d48c7bea222a97b27/C: creating writer at 1732317351227 (+408 ms)Flushing 72b8054778e76f5d48c7bea222a97b27/C: appending metadata at 1732317351230 (+3 ms)Flushing 72b8054778e76f5d48c7bea222a97b27/C: closing flushed file at 1732317351230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74f528b0: reopening flushed file at 1732317351640 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f0fae6: reopening flushed file at 1732317351646 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fd2563b: reopening flushed file at 1732317351651 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 72b8054778e76f5d48c7bea222a97b27 in 1252ms, sequenceid=754, compaction requested=true at 1732317351657 (+6 ms)Writing region close event to WAL at 1732317351775 (+118 ms)Running coprocessor post-close hooks at 1732317351779 (+4 ms)Closed at 1732317351779 2024-11-22T23:15:51,781 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:51,781 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=72b8054778e76f5d48c7bea222a97b27, regionState=CLOSED 2024-11-22T23:15:51,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:51,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-22T23:15:51,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 72b8054778e76f5d48c7bea222a97b27, server=a3d87a7361f3,46715,1732317251834 in 1.5370 sec 2024-11-22T23:15:51,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-22T23:15:51,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=72b8054778e76f5d48c7bea222a97b27, UNASSIGN in 1.5430 sec 2024-11-22T23:15:51,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-22T23:15:51,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5480 sec 2024-11-22T23:15:51,797 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317351797"}]},"ts":"1732317351797"} 2024-11-22T23:15:51,799 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:15:51,799 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:15:51,801 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5600 sec 2024-11-22T23:15:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-22T23:15:52,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:52,389 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:15:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,392 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=100, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-22T23:15:52,393 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=100, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,398 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:52,401 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/recovered.edits] 2024-11-22T23:15:52,405 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2c1b646a807045b98d6fe1ee8cbd84a3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/2c1b646a807045b98d6fe1ee8cbd84a3 2024-11-22T23:15:52,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3ca26682923640d3a593490dc1c7fc4f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/3ca26682923640d3a593490dc1c7fc4f 2024-11-22T23:15:52,408 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/788e7e707c6543e882539e4400573adb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/788e7e707c6543e882539e4400573adb 2024-11-22T23:15:52,409 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/99b3ec7c1f364ece9686e07c8a76f7de to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/A/99b3ec7c1f364ece9686e07c8a76f7de 2024-11-22T23:15:52,412 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0f823892db114e69bbef3d8e28b68f10 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/0f823892db114e69bbef3d8e28b68f10 2024-11-22T23:15:52,413 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/16bfffe3f24d4dafa61e0e4dfa6987ea to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/16bfffe3f24d4dafa61e0e4dfa6987ea 2024-11-22T23:15:52,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/39ae39af8456434bbe32772b78a0389e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/39ae39af8456434bbe32772b78a0389e 2024-11-22T23:15:52,415 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ec236126c9a498d83da73a0a6dfc091 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/B/5ec236126c9a498d83da73a0a6dfc091 2024-11-22T23:15:52,417 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4330eff1e2174853a07397adbb14da60 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/4330eff1e2174853a07397adbb14da60 2024-11-22T23:15:52,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/58f294b71310438d9f669fa9c3a0862b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/58f294b71310438d9f669fa9c3a0862b 2024-11-22T23:15:52,418 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/62b3254d5371419cb5a0eea2cb386388 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/62b3254d5371419cb5a0eea2cb386388 2024-11-22T23:15:52,419 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ae362501ccb24362af970e3f8c3961c3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/C/ae362501ccb24362af970e3f8c3961c3 2024-11-22T23:15:52,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/recovered.edits/757.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27/recovered.edits/757.seqid 2024-11-22T23:15:52,421 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/72b8054778e76f5d48c7bea222a97b27 2024-11-22T23:15:52,421 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:15:52,423 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=100, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,426 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:15:52,427 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:15:52,428 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=100, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,428 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:15:52,429 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317352428"}]},"ts":"9223372036854775807"} 2024-11-22T23:15:52,430 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:15:52,430 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 72b8054778e76f5d48c7bea222a97b27, NAME => 'TestAcidGuarantees,,1732317318003.72b8054778e76f5d48c7bea222a97b27.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:15:52,430 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:15:52,431 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317352430"}]},"ts":"9223372036854775807"} 2024-11-22T23:15:52,432 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:15:52,433 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=100, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-22T23:15:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-22T23:15:52,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:15:52,499 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:52,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=445 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=367 (was 344) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1002 (was 2027) 2024-11-22T23:15:52,521 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=367, ProcessCount=11, AvailableMemoryMB=1001 2024-11-22T23:15:52,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:15:52,522 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:15:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:52,524 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:15:52,524 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:52,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 101 2024-11-22T23:15:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:52,525 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:15:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742159_1335 (size=1061) 2024-11-22T23:15:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:52,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:15:52,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742160_1336 (size=53) 2024-11-22T23:15:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:53,337 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:15:53,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e92d0941565ecb9e195f60897027f996, disabling compactions & flushes 2024-11-22T23:15:53,338 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. after waiting 0 ms 2024-11-22T23:15:53,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,338 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,338 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e92d0941565ecb9e195f60897027f996: Waiting for close lock at 1732317353337Disabling compacts and flushes for region at 1732317353337Disabling writes for close at 1732317353338 (+1 ms)Writing region close event to WAL at 1732317353338Closed at 1732317353338 2024-11-22T23:15:53,339 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:15:53,339 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317353339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317353339"}]},"ts":"1732317353339"} 2024-11-22T23:15:53,340 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:15:53,341 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:15:53,342 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317353341"}]},"ts":"1732317353341"} 2024-11-22T23:15:53,343 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:15:53,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, ASSIGN}] 2024-11-22T23:15:53,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, ASSIGN 2024-11-22T23:15:53,345 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:15:53,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:53,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, ASSIGN because future has completed 2024-11-22T23:15:53,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:53,652 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,652 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7752): Opening region: {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:15:53,652 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,652 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:15:53,652 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7794): checking encryption for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,652 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7797): checking classloading for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,653 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,654 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:53,655 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName A 2024-11-22T23:15:53,655 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:53,655 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:53,655 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,656 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:53,656 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName B 2024-11-22T23:15:53,656 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:53,656 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:53,656 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,657 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:53,657 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName C 2024-11-22T23:15:53,657 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:53,658 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:53,658 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,658 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1038): replaying wal for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,658 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,659 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,659 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1048): stopping wal replay for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,659 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1060): Cleaning up temporary data for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,660 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:15:53,661 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): writing seq id for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,662 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:15:53,663 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1114): Opened e92d0941565ecb9e195f60897027f996; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68425613, jitterRate=0.019621089100837708}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:15:53,663 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:53,663 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1006): Region open journal for e92d0941565ecb9e195f60897027f996: Running coprocessor pre-open hook at 1732317353652Writing region info on filesystem at 1732317353653 (+1 ms)Initializing all the Stores at 1732317353653Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317353653Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317353653Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317353653Cleaning up temporary data from old regions at 1732317353659 (+6 ms)Running coprocessor post-open hooks at 1732317353663 (+4 ms)Region opened successfully at 1732317353663 2024-11-22T23:15:53,664 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., pid=103, masterSystemTime=1732317353649 2024-11-22T23:15:53,665 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,665 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:53,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:53,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:53,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-22T23:15:53,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 in 170 msec 2024-11-22T23:15:53,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-22T23:15:53,670 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, ASSIGN in 325 msec 2024-11-22T23:15:53,670 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:15:53,670 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317353670"}]},"ts":"1732317353670"} 2024-11-22T23:15:53,672 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:15:53,672 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:15:53,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-22T23:15:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-22T23:15:54,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:54,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:54,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:54,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:54,672 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e73fb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:54,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:54,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56cb9b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:54,677 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:54,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:54,681 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:15:54,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:15:54,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46b6b005 2024-11-22T23:15:54,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:15:54,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:15:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:15:54,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:15:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:15:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742161_1337 (size=1097) 2024-11-22T23:15:55,099 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:15:55,099 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:15:55,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:15:55,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE}] 2024-11-22T23:15:55,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE 2024-11-22T23:15:55,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,106 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE because future has completed 2024-11-22T23:15:55,106 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:15:55,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:55,259 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(122): Close e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,259 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:15:55,259 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1722): Closing e92d0941565ecb9e195f60897027f996, disabling compactions & flushes 2024-11-22T23:15:55,259 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,259 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,259 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. after waiting 0 ms 2024-11-22T23:15:55,259 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,263 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:15:55,263 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,263 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegion(1676): Region close journal for e92d0941565ecb9e195f60897027f996: Waiting for close lock at 1732317355259Running coprocessor pre-close hooks at 1732317355259Disabling compacts and flushes for region at 1732317355259Disabling writes for close at 1732317355259Writing region close event to WAL at 1732317355259Running coprocessor post-close hooks at 1732317355263 (+4 ms)Closed at 1732317355263 2024-11-22T23:15:55,263 WARN [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] regionserver.HRegionServer(3299): Not adding moved region record: e92d0941565ecb9e195f60897027f996 to self. 2024-11-22T23:15:55,265 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=107}] handler.UnassignRegionHandler(157): Closed e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,266 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=CLOSED 2024-11-22T23:15:55,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:55,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-22T23:15:55,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 in 163 msec 2024-11-22T23:15:55,271 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE; state=CLOSED, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=true 2024-11-22T23:15:55,421 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,458 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE because future has completed 2024-11-22T23:15:55,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:15:55,615 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,615 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,617 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,618 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:55,618 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName A 2024-11-22T23:15:55,619 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:55,620 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:55,621 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,622 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:55,622 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName B 2024-11-22T23:15:55,622 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:55,623 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:55,623 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,624 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:15:55,624 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e92d0941565ecb9e195f60897027f996 columnFamilyName C 2024-11-22T23:15:55,624 DEBUG [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:55,625 INFO [StoreOpener-e92d0941565ecb9e195f60897027f996-1 {}] regionserver.HStore(327): Store=e92d0941565ecb9e195f60897027f996/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:15:55,625 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,626 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,627 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,628 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,629 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,629 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,630 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:15:55,632 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,633 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened e92d0941565ecb9e195f60897027f996; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61947178, jitterRate=-0.07691511511802673}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:15:55,633 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,634 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for e92d0941565ecb9e195f60897027f996: Running coprocessor pre-open hook at 1732317355615Writing region info on filesystem at 1732317355615Initializing all the Stores at 1732317355616 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317355616Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317355617 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317355617Cleaning up temporary data from old regions at 1732317355629 (+12 ms)Running coprocessor post-open hooks at 1732317355633 (+4 ms)Region opened successfully at 1732317355634 (+1 ms) 2024-11-22T23:15:55,635 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., pid=108, masterSystemTime=1732317355612 2024-11-22T23:15:55,638 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,638 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,638 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=106 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=OPEN, openSeqNum=5, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=106, state=RUNNABLE, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:15:55,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=106 2024-11-22T23:15:55,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=106, state=SUCCESS, hasLock=false; OpenRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 in 181 msec 2024-11-22T23:15:55,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-22T23:15:55,643 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, REOPEN/MOVE in 539 msec 2024-11-22T23:15:55,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-22T23:15:55,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-22T23:15:55,647 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 960 msec 2024-11-22T23:15:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-22T23:15:55,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:55,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,649 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d56d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,651 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f0bc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,653 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446b2c41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a54569, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,656 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47520150, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,657 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632115fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1202173b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,659 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673cde56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,661 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5acd4312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,662 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@827f6c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,664 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60e10bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9056c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170f2068, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad4ce6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,670 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658db990, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aad5a5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685d1a7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,675 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4f1b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,676 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:15:55,677 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:15:55,677 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:15:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:15:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588ba4e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:15:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:15:55,679 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:15:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66bc461e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:15:55,680 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,680 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,680 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,680 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:55,680 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,681 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,681 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-22T23:15:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,682 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,683 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,684 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,684 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:15:55,684 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:55,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:15:55,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:15:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,691 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:15:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:55,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:15:55,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:55,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223ebf53cf68904eecabc6fc0fa110e9c3_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317355697/Put/seqid=0 2024-11-22T23:15:55,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742162_1338 (size=17034) 2024-11-22T23:15:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317415738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317415742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317415743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317415746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317415747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:55,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:55,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:55,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317415857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317415857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317415867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317415867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317415868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:55,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:55,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:55,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317416068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317416078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317416078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317416087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317416087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:56,133 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223ebf53cf68904eecabc6fc0fa110e9c3_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ebf53cf68904eecabc6fc0fa110e9c3_e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:56,134 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/a933247410b24c2db682ff9eb8e74498, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:15:56,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/a933247410b24c2db682ff9eb8e74498 is 175, key is test_row_0/A:col10/1732317355697/Put/seqid=0 2024-11-22T23:15:56,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742163_1339 (size=48139) 2024-11-22T23:15:56,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317416378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317416397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317416397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317416407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317416407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/a933247410b24c2db682ff9eb8e74498 2024-11-22T23:15:56,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2898c520585c4f54a8d178fb2a8fe0cf is 50, key is test_row_0/B:col10/1732317355697/Put/seqid=0 2024-11-22T23:15:56,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742164_1340 (size=12001) 2024-11-22T23:15:56,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:56,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317416887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:56,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317416907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317416907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317416928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317416928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:56,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:56,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2898c520585c4f54a8d178fb2a8fe0cf 2024-11-22T23:15:56,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/436cb6a240f942f1a79aaba3359cc779 is 50, key is test_row_0/C:col10/1732317355697/Put/seqid=0 2024-11-22T23:15:56,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742165_1341 (size=12001) 2024-11-22T23:15:57,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:57,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:15:57,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:57,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:57,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/436cb6a240f942f1a79aaba3359cc779 2024-11-22T23:15:57,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/a933247410b24c2db682ff9eb8e74498 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498 2024-11-22T23:15:57,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498, entries=250, sequenceid=16, filesize=47.0 K 2024-11-22T23:15:57,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2898c520585c4f54a8d178fb2a8fe0cf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf 2024-11-22T23:15:57,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T23:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/436cb6a240f942f1a79aaba3359cc779 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779 2024-11-22T23:15:57,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779, entries=150, sequenceid=16, filesize=11.7 K 2024-11-22T23:15:57,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e92d0941565ecb9e195f60897027f996 in 1699ms, sequenceid=16, compaction requested=false 2024-11-22T23:15:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:15:57,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:15:57,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:57,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e9370b9881074748b89dafdc52b1f829_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317355741/Put/seqid=0 2024-11-22T23:15:57,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742166_1342 (size=12154) 2024-11-22T23:15:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:57,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:57,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:15:57,955 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e9370b9881074748b89dafdc52b1f829_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e9370b9881074748b89dafdc52b1f829_e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:57,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/fcb4082b998742a0b288fa9bb06d526b, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:15:57,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/fcb4082b998742a0b288fa9bb06d526b is 175, key is test_row_0/A:col10/1732317355741/Put/seqid=0 2024-11-22T23:15:57,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317417946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317417948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317417948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:57,959 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in e92d0941565ecb9e195f60897027f996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,959 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in e92d0941565ecb9e195f60897027f996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742167_1343 (size=30955) 2024-11-22T23:15:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317417955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:57,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317417958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:57,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317418067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317418077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317418077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317418277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317418287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317418287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,370 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/fcb4082b998742a0b288fa9bb06d526b 2024-11-22T23:15:58,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/c0d1420c06a2427ca3e3e7ffa2a83920 is 50, key is test_row_0/B:col10/1732317355741/Put/seqid=0 2024-11-22T23:15:58,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742168_1344 (size=12001) 2024-11-22T23:15:58,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317418587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317418598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317418598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:58,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:58,783 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/c0d1420c06a2427ca3e3e7ffa2a83920 2024-11-22T23:15:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c3929eba80a74b9cb9ac6e037093f8b3 is 50, key is test_row_0/C:col10/1732317355741/Put/seqid=0 2024-11-22T23:15:58,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742169_1345 (size=12001) 2024-11-22T23:15:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317419097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317419118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317419118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:15:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:15:59,195 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c3929eba80a74b9cb9ac6e037093f8b3 2024-11-22T23:15:59,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/fcb4082b998742a0b288fa9bb06d526b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b 2024-11-22T23:15:59,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b, entries=150, sequenceid=36, filesize=30.2 K 2024-11-22T23:15:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/c0d1420c06a2427ca3e3e7ffa2a83920 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920 2024-11-22T23:15:59,208 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920, entries=150, sequenceid=36, filesize=11.7 K 2024-11-22T23:15:59,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c3929eba80a74b9cb9ac6e037093f8b3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3 2024-11-22T23:15:59,212 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3, entries=150, sequenceid=36, filesize=11.7 K 2024-11-22T23:15:59,213 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for e92d0941565ecb9e195f60897027f996 in 1692ms, sequenceid=36, compaction requested=false 2024-11-22T23:15:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:15:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:59,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-22T23:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-22T23:15:59,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-22T23:15:59,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5300 sec 2024-11-22T23:15:59,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 3.5360 sec 2024-11-22T23:15:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-22T23:15:59,837 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:15:59,841 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:15:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-22T23:15:59,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:15:59,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:15:59,844 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:15:59,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:15:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:15:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:15:59,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:15:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:15:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:15:59,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:59,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:15:59,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:15:59,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:15:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:15:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:15:59,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:15:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224ad0ac2bef0f4cc4915da99514deb757_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:00,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742170_1346 (size=14594) 2024-11-22T23:16:00,062 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:00,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411224ad0ac2bef0f4cc4915da99514deb757_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224ad0ac2bef0f4cc4915da99514deb757_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:00,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/01415962a30c41049aeb93109cdfa9f8, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:00,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/01415962a30c41049aeb93109cdfa9f8 is 175, key is test_row_0/A:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:00,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317420096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317420098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742171_1347 (size=39549) 2024-11-22T23:16:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317420118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,133 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in e92d0941565ecb9e195f60897027f996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317420140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317420140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,155 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in e92d0941565ecb9e195f60897027f996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,155 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in e92d0941565ecb9e195f60897027f996 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:16:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317420218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317420218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317420428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317420432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:16:00,531 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/01415962a30c41049aeb93109cdfa9f8 2024-11-22T23:16:00,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2437cf25ddfe40dd86e8ffbd10fe5bcf is 50, key is test_row_0/B:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742172_1348 (size=12001) 2024-11-22T23:16:00,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2437cf25ddfe40dd86e8ffbd10fe5bcf 2024-11-22T23:16:00,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c2c0ee5acd4b478ea2458f6e884cd973 is 50, key is test_row_0/C:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:00,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742173_1349 (size=12001) 2024-11-22T23:16:00,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c2c0ee5acd4b478ea2458f6e884cd973 2024-11-22T23:16:00,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/01415962a30c41049aeb93109cdfa9f8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8 2024-11-22T23:16:00,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8, entries=200, sequenceid=50, filesize=38.6 K 2024-11-22T23:16:00,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/2437cf25ddfe40dd86e8ffbd10fe5bcf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf 2024-11-22T23:16:00,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T23:16:00,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/c2c0ee5acd4b478ea2458f6e884cd973 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973 2024-11-22T23:16:00,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973, entries=150, sequenceid=50, filesize=11.7 K 2024-11-22T23:16:00,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e92d0941565ecb9e195f60897027f996 in 690ms, sequenceid=50, compaction requested=true 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:00,674 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:00,674 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:00,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:00,675 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:00,675 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:00,675 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,675 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=115.9 K 2024-11-22T23:16:00,675 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8] 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:00,676 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a933247410b24c2db682ff9eb8e74498, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732317355691 2024-11-22T23:16:00,676 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=35.2 K 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb4082b998742a0b288fa9bb06d526b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732317355740 2024-11-22T23:16:00,676 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2898c520585c4f54a8d178fb2a8fe0cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732317355694 2024-11-22T23:16:00,677 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01415962a30c41049aeb93109cdfa9f8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357938 2024-11-22T23:16:00,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d1420c06a2427ca3e3e7ffa2a83920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732317355740 2024-11-22T23:16:00,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2437cf25ddfe40dd86e8ffbd10fe5bcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357951 2024-11-22T23:16:00,685 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:00,687 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#306 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:00,687 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/b66c6a7b04ff43259f91ccdaf68b9c28 is 50, key is test_row_0/B:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:00,701 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411222f39ed929a03414fb6b10233751a7477_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:00,703 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411222f39ed929a03414fb6b10233751a7477_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:00,703 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222f39ed929a03414fb6b10233751a7477_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742174_1350 (size=12104) 2024-11-22T23:16:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:00,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:00,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:00,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227b3b7a3171784af9b19c8aa318f4b961_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317360756/Put/seqid=0 2024-11-22T23:16:00,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742175_1351 (size=4469) 2024-11-22T23:16:00,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742176_1352 (size=12154) 2024-11-22T23:16:00,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317420815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317420828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:00,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:00,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317420938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317420939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:00,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:16:01,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,136 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/b66c6a7b04ff43259f91ccdaf68b9c28 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b66c6a7b04ff43259f91ccdaf68b9c28 2024-11-22T23:16:01,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317421157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317421157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:01,173 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into b66c6a7b04ff43259f91ccdaf68b9c28(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:01,173 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:01,174 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=13, startTime=1732317360674; duration=0sec 2024-11-22T23:16:01,174 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:01,175 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:01,175 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:01,180 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:01,180 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:01,180 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,180 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#305 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:01,180 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=35.2 K 2024-11-22T23:16:01,181 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/1612b59603ed421b99a2b24d902e4bef is 175, key is test_row_0/A:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:01,183 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 436cb6a240f942f1a79aaba3359cc779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732317355694 2024-11-22T23:16:01,184 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c3929eba80a74b9cb9ac6e037093f8b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732317355740 2024-11-22T23:16:01,192 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c0ee5acd4b478ea2458f6e884cd973, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357951 2024-11-22T23:16:01,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:01,208 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227b3b7a3171784af9b19c8aa318f4b961_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b3b7a3171784af9b19c8aa318f4b961_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:01,209 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3387662f5f794f01a89c8aa6e271c690, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:01,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3387662f5f794f01a89c8aa6e271c690 is 175, key is test_row_0/A:col10/1732317360756/Put/seqid=0 2024-11-22T23:16:01,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742177_1353 (size=31058) 2024-11-22T23:16:01,218 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:01,219 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3a725808e40f4c37a9b7c98c35e94e96 is 50, key is test_row_0/C:col10/1732317359983/Put/seqid=0 2024-11-22T23:16:01,223 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/1612b59603ed421b99a2b24d902e4bef as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef 2024-11-22T23:16:01,240 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into 1612b59603ed421b99a2b24d902e4bef(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:01,240 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:01,240 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=13, startTime=1732317360674; duration=0sec 2024-11-22T23:16:01,241 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:01,241 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:01,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:01,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742178_1354 (size=30955) 2024-11-22T23:16:01,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3387662f5f794f01a89c8aa6e271c690 2024-11-22T23:16:01,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742179_1355 (size=12104) 2024-11-22T23:16:01,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/8c84bd27fdea4e8faad0c494edca2ff1 is 50, key is test_row_0/B:col10/1732317360756/Put/seqid=0 2024-11-22T23:16:01,293 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3a725808e40f4c37a9b7c98c35e94e96 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3a725808e40f4c37a9b7c98c35e94e96 2024-11-22T23:16:01,299 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into 3a725808e40f4c37a9b7c98c35e94e96(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:01,299 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:01,299 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=13, startTime=1732317360674; duration=0sec 2024-11-22T23:16:01,299 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:01,299 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:01,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742180_1356 (size=12001) 2024-11-22T23:16:01,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317421477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:01,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317421477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:01,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:01,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:01,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/8c84bd27fdea4e8faad0c494edca2ff1 2024-11-22T23:16:01,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/d8977d94e73c4087b937b2b213af31ef is 50, key is test_row_0/C:col10/1732317360756/Put/seqid=0 2024-11-22T23:16:01,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742181_1357 (size=12001) 2024-11-22T23:16:01,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/d8977d94e73c4087b937b2b213af31ef 2024-11-22T23:16:01,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3387662f5f794f01a89c8aa6e271c690 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690 2024-11-22T23:16:01,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690, entries=150, sequenceid=69, filesize=30.2 K 2024-11-22T23:16:01,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/8c84bd27fdea4e8faad0c494edca2ff1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1 2024-11-22T23:16:01,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1, entries=150, sequenceid=69, filesize=11.7 K 2024-11-22T23:16:01,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/d8977d94e73c4087b937b2b213af31ef as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef 2024-11-22T23:16:01,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef, entries=150, sequenceid=69, filesize=11.7 K 2024-11-22T23:16:01,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e92d0941565ecb9e195f60897027f996 in 1067ms, sequenceid=69, compaction requested=false 2024-11-22T23:16:01,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:01,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-22T23:16:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:01,863 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:01,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:01,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226824e65d27f84450981a6a1ee8421a8a_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317360812/Put/seqid=0 2024-11-22T23:16:01,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742182_1358 (size=12154) 2024-11-22T23:16:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:01,931 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411226824e65d27f84450981a6a1ee8421a8a_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226824e65d27f84450981a6a1ee8421a8a_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/7ac97bac05414e32a3184afe63979f89, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:01,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/7ac97bac05414e32a3184afe63979f89 is 175, key is test_row_0/A:col10/1732317360812/Put/seqid=0 2024-11-22T23:16:01,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742183_1359 (size=30955) 2024-11-22T23:16:01,973 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/7ac97bac05414e32a3184afe63979f89 2024-11-22T23:16:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:16:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/f634dda22ac04e8e985c9fee6b988ad1 is 50, key is test_row_0/B:col10/1732317360812/Put/seqid=0 2024-11-22T23:16:02,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:02,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742184_1360 (size=12001) 2024-11-22T23:16:02,057 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/f634dda22ac04e8e985c9fee6b988ad1 2024-11-22T23:16:02,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/44b68562929748b192555ddccf51da59 is 50, key is test_row_0/C:col10/1732317360812/Put/seqid=0 2024-11-22T23:16:02,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742185_1361 (size=12001) 2024-11-22T23:16:02,127 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/44b68562929748b192555ddccf51da59 2024-11-22T23:16:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/7ac97bac05414e32a3184afe63979f89 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89 2024-11-22T23:16:02,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89, entries=150, sequenceid=85, filesize=30.2 K 2024-11-22T23:16:02,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/f634dda22ac04e8e985c9fee6b988ad1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1 2024-11-22T23:16:02,152 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1, entries=150, sequenceid=85, filesize=11.7 K 2024-11-22T23:16:02,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/44b68562929748b192555ddccf51da59 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59 2024-11-22T23:16:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59, entries=150, sequenceid=85, filesize=11.7 K 2024-11-22T23:16:02,161 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=93.93 KB/96180 for e92d0941565ecb9e195f60897027f996 in 298ms, sequenceid=85, compaction requested=true 2024-11-22T23:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:02,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-22T23:16:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-22T23:16:02,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-22T23:16:02,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-22T23:16:02,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.3240 sec 2024-11-22T23:16:02,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:02,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:02,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220f1d3e0d643b4dc285d41ea4cf47ec61_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:02,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317422201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317422217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317422219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317422226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317422229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742186_1362 (size=14594) 2024-11-22T23:16:02,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317422337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317422347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317422347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317422347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317422358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,375 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317422558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317422568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317422569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317422569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317422578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:02,685 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220f1d3e0d643b4dc285d41ea4cf47ec61_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220f1d3e0d643b4dc285d41ea4cf47ec61_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:02,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6cc6aa0d25d84d89a61121ca2c842617, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:02,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6cc6aa0d25d84d89a61121ca2c842617 is 175, key is test_row_0/A:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:02,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742187_1363 (size=39549) 2024-11-22T23:16:02,702 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6cc6aa0d25d84d89a61121ca2c842617 2024-11-22T23:16:02,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/0d60a75bba124eb59e10c4a98dd8b7ba is 50, key is test_row_0/B:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:02,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742188_1364 (size=12001) 2024-11-22T23:16:02,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/0d60a75bba124eb59e10c4a98dd8b7ba 2024-11-22T23:16:02,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/83242290f400440caec7edc5dbd6a3f6 is 50, key is test_row_0/C:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:02,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742189_1365 (size=12001) 2024-11-22T23:16:02,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/83242290f400440caec7edc5dbd6a3f6 2024-11-22T23:16:02,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6cc6aa0d25d84d89a61121ca2c842617 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617 2024-11-22T23:16:02,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617, entries=200, sequenceid=103, filesize=38.6 K 2024-11-22T23:16:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/0d60a75bba124eb59e10c4a98dd8b7ba as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba 2024-11-22T23:16:02,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba, entries=150, sequenceid=103, filesize=11.7 K 2024-11-22T23:16:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/83242290f400440caec7edc5dbd6a3f6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6 2024-11-22T23:16:02,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6, entries=150, sequenceid=103, filesize=11.7 K 2024-11-22T23:16:02,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e92d0941565ecb9e195f60897027f996 in 644ms, sequenceid=103, compaction requested=true 2024-11-22T23:16:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:02,814 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:02,814 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:02,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:02,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:02,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,816 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:02,816 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:02,816 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:02,816 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=129.4 K 2024-11-22T23:16:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,816 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:02,816 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617] 2024-11-22T23:16:02,816 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:02,816 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:02,817 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:02,817 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b66c6a7b04ff43259f91ccdaf68b9c28, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=47.0 K 2024-11-22T23:16:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,817 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b66c6a7b04ff43259f91ccdaf68b9c28, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357951 2024-11-22T23:16:02,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,817 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1612b59603ed421b99a2b24d902e4bef, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357951 2024-11-22T23:16:02,817 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c84bd27fdea4e8faad0c494edca2ff1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732317360069 2024-11-22T23:16:02,818 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3387662f5f794f01a89c8aa6e271c690, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732317360069 2024-11-22T23:16:02,818 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f634dda22ac04e8e985c9fee6b988ad1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732317360807 2024-11-22T23:16:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,818 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ac97bac05414e32a3184afe63979f89, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732317360807 2024-11-22T23:16:02,818 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d60a75bba124eb59e10c4a98dd8b7ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:02,818 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cc6aa0d25d84d89a61121ca2c842617, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:02,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,827 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,829 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,829 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/0029f6e8b1e849bbb375d88bf00d79f7 is 50, key is test_row_0/B:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,841 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122c7a713373ab54aeebe98168a420c65db_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,843 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122c7a713373ab54aeebe98168a420c65db_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,844 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c7a713373ab54aeebe98168a420c65db_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742190_1366 (size=12241) 2024-11-22T23:16:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742191_1367 (size=4469) 2024-11-22T23:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:02,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:02,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f92b6e57c5664526bcb41e7ba5d16d4d_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317362929/Put/seqid=0 2024-11-22T23:16:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742192_1368 (size=12154) 2024-11-22T23:16:02,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:02,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317422985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317422991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317422992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317422997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317422998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317423108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317423108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317423108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317423117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317423128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,275 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/0029f6e8b1e849bbb375d88bf00d79f7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0029f6e8b1e849bbb375d88bf00d79f7 2024-11-22T23:16:03,279 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into 0029f6e8b1e849bbb375d88bf00d79f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:03,280 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:03,280 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=12, startTime=1732317362814; duration=0sec 2024-11-22T23:16:03,280 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:03,280 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:03,280 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:03,281 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:03,281 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:03,281 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:03,282 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3a725808e40f4c37a9b7c98c35e94e96, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=47.0 K 2024-11-22T23:16:03,282 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a725808e40f4c37a9b7c98c35e94e96, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732317357951 2024-11-22T23:16:03,282 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d8977d94e73c4087b937b2b213af31ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732317360069 2024-11-22T23:16:03,283 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b68562929748b192555ddccf51da59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732317360807 2024-11-22T23:16:03,283 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 83242290f400440caec7edc5dbd6a3f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:03,285 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#317 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:03,285 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/035ab2c29c6c4094b5a6f74f8d09c960 is 175, key is test_row_0/A:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:03,316 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#320 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:03,317 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/f8112af0bee04fe4ae0cf996bea50781 is 50, key is test_row_0/C:col10/1732317362061/Put/seqid=0 2024-11-22T23:16:03,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317423327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317423327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317423327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742193_1369 (size=31195) 2024-11-22T23:16:03,346 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/035ab2c29c6c4094b5a6f74f8d09c960 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960 2024-11-22T23:16:03,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742194_1370 (size=12241) 2024-11-22T23:16:03,357 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into 035ab2c29c6c4094b5a6f74f8d09c960(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:03,357 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,357 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=12, startTime=1732317362814; duration=0sec 2024-11-22T23:16:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317423338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,357 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:03,357 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:03,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317423347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,364 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:03,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122f92b6e57c5664526bcb41e7ba5d16d4d_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f92b6e57c5664526bcb41e7ba5d16d4d_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:03,373 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c8c9b628e78743f3a841443f62f1e41a, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:03,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c8c9b628e78743f3a841443f62f1e41a is 175, key is test_row_0/A:col10/1732317362929/Put/seqid=0 2024-11-22T23:16:03,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742195_1371 (size=30951) 2024-11-22T23:16:03,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c8c9b628e78743f3a841443f62f1e41a 2024-11-22T23:16:03,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/858afd1169c8463eaa76f51f3a2e11e7 is 50, key is test_row_0/B:col10/1732317362929/Put/seqid=0 2024-11-22T23:16:03,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742196_1372 (size=9657) 2024-11-22T23:16:03,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/858afd1169c8463eaa76f51f3a2e11e7 2024-11-22T23:16:03,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/5598a65b5e044b5095ba650a4f2a2627 is 50, key is test_row_0/C:col10/1732317362929/Put/seqid=0 2024-11-22T23:16:03,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742197_1373 (size=9657) 2024-11-22T23:16:03,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317423647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317423648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317423648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317423667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317423667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:03,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:03,759 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/f8112af0bee04fe4ae0cf996bea50781 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f8112af0bee04fe4ae0cf996bea50781 2024-11-22T23:16:03,763 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into f8112af0bee04fe4ae0cf996bea50781(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:03,763 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:03,764 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=12, startTime=1732317362814; duration=0sec 2024-11-22T23:16:03,764 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:03,764 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:03,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/5598a65b5e044b5095ba650a4f2a2627 2024-11-22T23:16:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c8c9b628e78743f3a841443f62f1e41a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a 2024-11-22T23:16:03,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a, entries=150, sequenceid=118, filesize=30.2 K 2024-11-22T23:16:03,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/858afd1169c8463eaa76f51f3a2e11e7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7 2024-11-22T23:16:03,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7, entries=100, sequenceid=118, filesize=9.4 K 2024-11-22T23:16:03,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/5598a65b5e044b5095ba650a4f2a2627 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627 2024-11-22T23:16:03,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627, entries=100, sequenceid=118, filesize=9.4 K 2024-11-22T23:16:03,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e92d0941565ecb9e195f60897027f996 in 968ms, sequenceid=118, compaction requested=false 2024-11-22T23:16:03,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:04,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-22T23:16:04,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:04,017 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-22T23:16:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:04,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:04,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:04,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:04,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-22T23:16:04,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:04,177 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-22T23:16:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:04,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:04,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:04,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228dc7a2b5a7d6453b8b2779451b08473f_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317362991/Put/seqid=0 2024-11-22T23:16:04,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742198_1374 (size=14794) 2024-11-22T23:16:04,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:04,207 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411228dc7a2b5a7d6453b8b2779451b08473f_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dc7a2b5a7d6453b8b2779451b08473f_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/07d7089997c347ee9f783d7a4aa802cd, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:04,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/07d7089997c347ee9f783d7a4aa802cd is 175, key is test_row_0/A:col10/1732317362991/Put/seqid=0 2024-11-22T23:16:04,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742199_1375 (size=39749) 2024-11-22T23:16:04,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317424256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317424272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317424282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317424286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317424292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:04,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317424397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317424408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317424408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317424418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,622 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/07d7089997c347ee9f783d7a4aa802cd 2024-11-22T23:16:04,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317424617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/ef5a1d8bc3e940219a310b6a3e2d5d7d is 50, key is test_row_0/B:col10/1732317362991/Put/seqid=0 2024-11-22T23:16:04,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742200_1376 (size=12151) 2024-11-22T23:16:04,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317424628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317424628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317424637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317424637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:04,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317424938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317424947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317424947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317424957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317424957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:04,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,032 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/ef5a1d8bc3e940219a310b6a3e2d5d7d 2024-11-22T23:16:05,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/f1207ea3870447a1bba9781c86da6dec is 50, key is test_row_0/C:col10/1732317362991/Put/seqid=0 2024-11-22T23:16:05,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742201_1377 (size=12151) 2024-11-22T23:16:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:05,446 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/f1207ea3870447a1bba9781c86da6dec 2024-11-22T23:16:05,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/07d7089997c347ee9f783d7a4aa802cd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd 2024-11-22T23:16:05,456 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd, entries=200, sequenceid=139, filesize=38.8 K 2024-11-22T23:16:05,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/ef5a1d8bc3e940219a310b6a3e2d5d7d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d 2024-11-22T23:16:05,461 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T23:16:05,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/f1207ea3870447a1bba9781c86da6dec as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec 2024-11-22T23:16:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317425458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,468 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec, entries=150, sequenceid=139, filesize=11.9 K 2024-11-22T23:16:05,469 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e92d0941565ecb9e195f60897027f996 in 1293ms, sequenceid=139, compaction requested=true 2024-11-22T23:16:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:05,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-22T23:16:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-22T23:16:05,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-22T23:16:05,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-11-22T23:16:05,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4570 sec 2024-11-22T23:16:05,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-22T23:16:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:05,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:05,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:05,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220475b70740a54a7aaba7b1a03bd99664_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:05,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742202_1378 (size=17284) 2024-11-22T23:16:05,534 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:05,546 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411220475b70740a54a7aaba7b1a03bd99664_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220475b70740a54a7aaba7b1a03bd99664_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:05,548 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/087fb847fbf54f07aa612b6a08a77437, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:05,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/087fb847fbf54f07aa612b6a08a77437 is 175, key is test_row_0/A:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:05,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742203_1379 (size=48389) 2024-11-22T23:16:05,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317425566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317425577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317425574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317425578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317425688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317425708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317425708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317425709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317425918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317425918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317425927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317425937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:05,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:05,959 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/087fb847fbf54f07aa612b6a08a77437 2024-11-22T23:16:05,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3e7e0b7fec874e69b3da66c510e487c0 is 50, key is test_row_0/B:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:05,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742204_1380 (size=12151) 2024-11-22T23:16:05,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3e7e0b7fec874e69b3da66c510e487c0 2024-11-22T23:16:05,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/a077994232d24bed9b3dee1d44a068e4 is 50, key is test_row_0/C:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:05,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742205_1381 (size=12151) 2024-11-22T23:16:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-22T23:16:06,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:06,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:06,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-22T23:16:06,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:06,170 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:06,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317426228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317426229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317426248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317426248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:06,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-22T23:16:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:06,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:06,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:06,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/a077994232d24bed9b3dee1d44a068e4 2024-11-22T23:16:06,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/087fb847fbf54f07aa612b6a08a77437 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437 2024-11-22T23:16:06,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437, entries=250, sequenceid=156, filesize=47.3 K 2024-11-22T23:16:06,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3e7e0b7fec874e69b3da66c510e487c0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0 2024-11-22T23:16:06,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T23:16:06,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/a077994232d24bed9b3dee1d44a068e4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4 2024-11-22T23:16:06,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4, entries=150, sequenceid=156, filesize=11.9 K 2024-11-22T23:16:06,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for e92d0941565ecb9e195f60897027f996 in 930ms, sequenceid=156, compaction requested=true 2024-11-22T23:16:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:06,409 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:06,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:06,409 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:06,410 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150284 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:06,410 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:06,410 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:06,410 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:06,410 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,410 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:06,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:06,410 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=146.8 K 2024-11-22T23:16:06,410 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0029f6e8b1e849bbb375d88bf00d79f7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=45.1 K 2024-11-22T23:16:06,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:06,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:06,410 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,410 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437] 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035ab2c29c6c4094b5a6f74f8d09c960, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0029f6e8b1e849bbb375d88bf00d79f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 858afd1169c8463eaa76f51f3a2e11e7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732317362196 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8c9b628e78743f3a841443f62f1e41a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732317362196 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ef5a1d8bc3e940219a310b6a3e2d5d7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317362991 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d7089997c347ee9f783d7a4aa802cd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317362983 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7e0b7fec874e69b3da66c510e487c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:06,411 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087fb847fbf54f07aa612b6a08a77437, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:06,417 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:06,419 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122691f74ac527d40558abcf2730c16f2a6_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:06,419 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:06,420 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/17adb30000ae41368e8c5ff9ca72645c is 50, key is test_row_0/B:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:06,422 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122691f74ac527d40558abcf2730c16f2a6_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:06,422 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122691f74ac527d40558abcf2730c16f2a6_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:06,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742206_1382 (size=12527) 2024-11-22T23:16:06,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742207_1383 (size=4469) 2024-11-22T23:16:06,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,475 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:06,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:06,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122494961775f694d498048d8a114714ac0_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317365576/Put/seqid=0 2024-11-22T23:16:06,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:06,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:06,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742208_1384 (size=12304) 2024-11-22T23:16:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317426592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317426708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317426747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317426747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317426767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317426767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:06,830 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/17adb30000ae41368e8c5ff9ca72645c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/17adb30000ae41368e8c5ff9ca72645c 2024-11-22T23:16:06,834 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into 17adb30000ae41368e8c5ff9ca72645c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:06,834 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:06,834 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=12, startTime=1732317366409; duration=0sec 2024-11-22T23:16:06,834 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:06,834 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:06,834 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:06,835 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:06,835 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:06,835 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:06,836 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f8112af0bee04fe4ae0cf996bea50781, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=45.1 K 2024-11-22T23:16:06,837 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f8112af0bee04fe4ae0cf996bea50781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732317362061 2024-11-22T23:16:06,837 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 5598a65b5e044b5095ba650a4f2a2627, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732317362196 2024-11-22T23:16:06,837 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#329 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:06,837 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f1207ea3870447a1bba9781c86da6dec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732317362991 2024-11-22T23:16:06,838 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/477c1156f638422db6a9c354fe0c6564 is 175, key is test_row_0/A:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:06,839 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a077994232d24bed9b3dee1d44a068e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:06,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742209_1385 (size=31481) 2024-11-22T23:16:06,859 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:06,859 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/76d7ac3b05d34d9f85cd3c306b43d817 is 50, key is test_row_0/C:col10/1732317364212/Put/seqid=0 2024-11-22T23:16:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742210_1386 (size=12527) 2024-11-22T23:16:06,875 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/76d7ac3b05d34d9f85cd3c306b43d817 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/76d7ac3b05d34d9f85cd3c306b43d817 2024-11-22T23:16:06,880 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into 76d7ac3b05d34d9f85cd3c306b43d817(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:06,880 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:06,880 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=12, startTime=1732317366410; duration=0sec 2024-11-22T23:16:06,880 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:06,880 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:06,897 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122494961775f694d498048d8a114714ac0_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122494961775f694d498048d8a114714ac0_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:06,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/66482da0ef344e679e0bd3eb1c6986dc, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:06,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/66482da0ef344e679e0bd3eb1c6986dc is 175, key is test_row_0/A:col10/1732317365576/Put/seqid=0 2024-11-22T23:16:06,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742211_1387 (size=31105) 2024-11-22T23:16:06,912 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/66482da0ef344e679e0bd3eb1c6986dc 2024-11-22T23:16:06,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/6b10a0bd64a4490daf4dfb1ee2ebe863 is 50, key is test_row_0/B:col10/1732317365576/Put/seqid=0 2024-11-22T23:16:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742212_1388 (size=12151) 2024-11-22T23:16:06,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317426927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317427247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,259 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/477c1156f638422db6a9c354fe0c6564 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564 2024-11-22T23:16:07,263 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into 477c1156f638422db6a9c354fe0c6564(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:07,263 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:07,263 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=12, startTime=1732317366409; duration=0sec 2024-11-22T23:16:07,263 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:07,263 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:07,334 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/6b10a0bd64a4490daf4dfb1ee2ebe863 2024-11-22T23:16:07,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/929abba400fd423bb90de0759b99946b is 50, key is test_row_0/C:col10/1732317365576/Put/seqid=0 2024-11-22T23:16:07,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742213_1389 (size=12151) 2024-11-22T23:16:07,744 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/929abba400fd423bb90de0759b99946b 2024-11-22T23:16:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/66482da0ef344e679e0bd3eb1c6986dc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc 2024-11-22T23:16:07,750 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc, entries=150, sequenceid=171, filesize=30.4 K 2024-11-22T23:16:07,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/6b10a0bd64a4490daf4dfb1ee2ebe863 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863 2024-11-22T23:16:07,758 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863, entries=150, sequenceid=171, filesize=11.9 K 2024-11-22T23:16:07,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/929abba400fd423bb90de0759b99946b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b 2024-11-22T23:16:07,764 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-22T23:16:07,765 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e92d0941565ecb9e195f60897027f996 in 1290ms, sequenceid=171, compaction requested=false 2024-11-22T23:16:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-22T23:16:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-22T23:16:07,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-22T23:16:07,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-22T23:16:07,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.6000 sec 2024-11-22T23:16:07,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:07,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-22T23:16:07,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:07,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:07,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:07,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:07,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:07,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:07,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232b55210e09f45b4a8a0d500630487c7_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:07,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742214_1390 (size=14794) 2024-11-22T23:16:07,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317427795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317427798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317427799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317427800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317427801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317427917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317427917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317427917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317427917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:07,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317427917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:07,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317428127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,201 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:08,204 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112232b55210e09f45b4a8a0d500630487c7_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232b55210e09f45b4a8a0d500630487c7_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:08,204 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/5068a07763cf43c9b9e06879e7616736, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:08,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/5068a07763cf43c9b9e06879e7616736 is 175, key is test_row_0/A:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:08,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742215_1391 (size=39749) 2024-11-22T23:16:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-22T23:16:08,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:08,318 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-22T23:16:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:08,319 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:08,320 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:08,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:08,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317428437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317428448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317428448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317428448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317428448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/5068a07763cf43c9b9e06879e7616736 2024-11-22T23:16:08,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/aa3ef8aebaad48d3afbe32aa1efff846 is 50, key is test_row_0/B:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:08,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:08,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742216_1392 (size=12151) 2024-11-22T23:16:08,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:08,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:08,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:08,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317428947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317428958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317428960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317428960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:08,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317428968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:08,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:09,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/aa3ef8aebaad48d3afbe32aa1efff846 2024-11-22T23:16:09,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b21dafc002fe468c965d15853e4d7f4a is 50, key is test_row_0/C:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:09,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742217_1393 (size=12151) 2024-11-22T23:16:09,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:09,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b21dafc002fe468c965d15853e4d7f4a 2024-11-22T23:16:09,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/5068a07763cf43c9b9e06879e7616736 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736 2024-11-22T23:16:09,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736, entries=200, sequenceid=192, filesize=38.8 K 2024-11-22T23:16:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/aa3ef8aebaad48d3afbe32aa1efff846 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846 2024-11-22T23:16:09,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846, entries=150, sequenceid=192, filesize=11.9 K 2024-11-22T23:16:09,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b21dafc002fe468c965d15853e4d7f4a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a 2024-11-22T23:16:09,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-22T23:16:09,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e92d0941565ecb9e195f60897027f996 in 1716ms, sequenceid=192, compaction requested=true 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:09,488 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:09,488 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:09,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:09,491 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,491 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,491 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/17adb30000ae41368e8c5ff9ca72645c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=36.0 K 2024-11-22T23:16:09,491 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=99.9 K 2024-11-22T23:16:09,491 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736] 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 17adb30000ae41368e8c5ff9ca72645c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477c1156f638422db6a9c354fe0c6564, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66482da0ef344e679e0bd3eb1c6986dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732317365556 2024-11-22T23:16:09,491 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b10a0bd64a4490daf4dfb1ee2ebe863, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732317365556 2024-11-22T23:16:09,492 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5068a07763cf43c9b9e06879e7616736, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:09,492 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3ef8aebaad48d3afbe32aa1efff846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:09,495 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:09,497 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122126a1b1495944c6a99aa665d4908de84_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:09,497 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:09,498 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/b21878e13ea3458fb4cd2980946ea69b is 50, key is test_row_0/B:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:09,498 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122126a1b1495944c6a99aa665d4908de84_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:09,498 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122126a1b1495944c6a99aa665d4908de84_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:09,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742218_1394 (size=12629) 2024-11-22T23:16:09,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742219_1395 (size=4469) 2024-11-22T23:16:09,509 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#338 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:09,510 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c78fc86c486a4068adf436615379e5dd is 175, key is test_row_0/A:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:09,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742220_1396 (size=31583) 2024-11-22T23:16:09,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-22T23:16:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,552 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122578b99b5b3734fcd9bad5efc3b3c62bc_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317367800/Put/seqid=0 2024-11-22T23:16:09,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742221_1397 (size=12304) 2024-11-22T23:16:09,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:09,599 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122578b99b5b3734fcd9bad5efc3b3c62bc_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122578b99b5b3734fcd9bad5efc3b3c62bc_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/8cacffc9828a413ba7c9a99cdc43b282, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:09,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/8cacffc9828a413ba7c9a99cdc43b282 is 175, key is test_row_0/A:col10/1732317367800/Put/seqid=0 2024-11-22T23:16:09,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742222_1398 (size=31105) 2024-11-22T23:16:09,807 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:16:09,908 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/b21878e13ea3458fb4cd2980946ea69b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b21878e13ea3458fb4cd2980946ea69b 2024-11-22T23:16:09,913 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into b21878e13ea3458fb4cd2980946ea69b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:09,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:09,913 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=13, startTime=1732317369488; duration=0sec 2024-11-22T23:16:09,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:09,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:09,913 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:09,914 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:09,914 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:09,914 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:09,914 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/76d7ac3b05d34d9f85cd3c306b43d817, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=36.0 K 2024-11-22T23:16:09,914 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d7ac3b05d34d9f85cd3c306b43d817, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732317364212 2024-11-22T23:16:09,915 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 929abba400fd423bb90de0759b99946b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732317365556 2024-11-22T23:16:09,915 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b21dafc002fe468c965d15853e4d7f4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:09,922 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:09,923 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/87cd15acbbad43229d42ff0e0b3aeab4 is 50, key is test_row_0/C:col10/1732317366585/Put/seqid=0 2024-11-22T23:16:09,927 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c78fc86c486a4068adf436615379e5dd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd 2024-11-22T23:16:09,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742223_1399 (size=12629) 2024-11-22T23:16:09,938 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into c78fc86c486a4068adf436615379e5dd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:09,938 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:09,938 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=13, startTime=1732317369488; duration=0sec 2024-11-22T23:16:09,938 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:09,938 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:09,942 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/87cd15acbbad43229d42ff0e0b3aeab4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/87cd15acbbad43229d42ff0e0b3aeab4 2024-11-22T23:16:09,951 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into 87cd15acbbad43229d42ff0e0b3aeab4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:09,951 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:09,951 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=13, startTime=1732317369488; duration=0sec 2024-11-22T23:16:09,951 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:09,951 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:09,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:10,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317429998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317430007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317430008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,015 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/8cacffc9828a413ba7c9a99cdc43b282 2024-11-22T23:16:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317430009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317430009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/a2f9a69a3507444996d4d6267271edcd is 50, key is test_row_0/B:col10/1732317367800/Put/seqid=0 2024-11-22T23:16:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742224_1400 (size=12151) 2024-11-22T23:16:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317430117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317430118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317430118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317430127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317430127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317430338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317430338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317430338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317430347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,427 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/a2f9a69a3507444996d4d6267271edcd 2024-11-22T23:16:10,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/9415cda32360424e969d7a06490bd25b is 50, key is test_row_0/C:col10/1732317367800/Put/seqid=0 2024-11-22T23:16:10,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742225_1401 (size=12151) 2024-11-22T23:16:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:10,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317430658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317430658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317430658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317430658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317430658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:10,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:10,839 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/9415cda32360424e969d7a06490bd25b 2024-11-22T23:16:10,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/8cacffc9828a413ba7c9a99cdc43b282 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282 2024-11-22T23:16:10,845 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282, entries=150, sequenceid=206, filesize=30.4 K 2024-11-22T23:16:10,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/a2f9a69a3507444996d4d6267271edcd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd 2024-11-22T23:16:10,848 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T23:16:10,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/9415cda32360424e969d7a06490bd25b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b 2024-11-22T23:16:10,852 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b, entries=150, sequenceid=206, filesize=11.9 K 2024-11-22T23:16:10,853 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e92d0941565ecb9e195f60897027f996 in 1302ms, sequenceid=206, compaction requested=false 2024-11-22T23:16:10,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:10,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:10,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-22T23:16:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-22T23:16:10,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-22T23:16:10,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-22T23:16:10,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5380 sec 2024-11-22T23:16:11,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:16:11,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:11,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:11,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c273d3fbdbff45baaa01999b9e4fd0e9_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317431202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317431203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317431215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317431215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317431218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742226_1402 (size=17284) 2024-11-22T23:16:11,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:11,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c273d3fbdbff45baaa01999b9e4fd0e9_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c273d3fbdbff45baaa01999b9e4fd0e9_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:11,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/dcf67f2978a4429488bef7c9afe416fc, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/dcf67f2978a4429488bef7c9afe416fc is 175, key is test_row_0/A:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742227_1403 (size=48389) 2024-11-22T23:16:11,279 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/dcf67f2978a4429488bef7c9afe416fc 2024-11-22T23:16:11,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/d584750119ac4568aec440a29b7e52c5 is 50, key is test_row_0/B:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742228_1404 (size=12151) 2024-11-22T23:16:11,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/d584750119ac4568aec440a29b7e52c5 2024-11-22T23:16:11,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317431327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317431327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/1d10ce1947f948b383dbef30cc32d131 is 50, key is test_row_0/C:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742229_1405 (size=12151) 2024-11-22T23:16:11,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/1d10ce1947f948b383dbef30cc32d131 2024-11-22T23:16:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317431348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317431349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/dcf67f2978a4429488bef7c9afe416fc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc 2024-11-22T23:16:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc, entries=250, sequenceid=230, filesize=47.3 K 2024-11-22T23:16:11,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/d584750119ac4568aec440a29b7e52c5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5 2024-11-22T23:16:11,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317431350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:16:11,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/1d10ce1947f948b383dbef30cc32d131 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131 2024-11-22T23:16:11,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131, entries=150, sequenceid=230, filesize=11.9 K 2024-11-22T23:16:11,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e92d0941565ecb9e195f60897027f996 in 192ms, sequenceid=230, compaction requested=true 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:11,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:11,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:11,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:11,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111077 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:11,371 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:11,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:11,371 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:11,371 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:11,371 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:11,371 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=108.5 K 2024-11-22T23:16:11,371 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b21878e13ea3458fb4cd2980946ea69b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=36.1 K 2024-11-22T23:16:11,371 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:11,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc] 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b21878e13ea3458fb4cd2980946ea69b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c78fc86c486a4068adf436615379e5dd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f9a69a3507444996d4d6267271edcd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732317367791 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cacffc9828a413ba7c9a99cdc43b282, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732317367791 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d584750119ac4568aec440a29b7e52c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370007 2024-11-22T23:16:11,372 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf67f2978a4429488bef7c9afe416fc, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370006 2024-11-22T23:16:11,382 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:11,383 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/c84caf2fa67b4859b729322269293ce5 is 50, key is test_row_0/B:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,395 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,405 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122c8caf7bd72e14bcc80ac169281c87893_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,407 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122c8caf7bd72e14bcc80ac169281c87893_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,407 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c8caf7bd72e14bcc80ac169281c87893_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742230_1406 (size=12731) 2024-11-22T23:16:11,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742231_1407 (size=4469) 2024-11-22T23:16:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:11,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:11,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:11,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5d60ab5a5584ab59f73a87db190674b_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317371215/Put/seqid=0 2024-11-22T23:16:11,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742232_1408 (size=14794) 2024-11-22T23:16:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317431594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317431595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317431596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317431598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317431603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317431708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317431717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317431717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317431717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317431717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,814 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/c84caf2fa67b4859b729322269293ce5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c84caf2fa67b4859b729322269293ce5 2024-11-22T23:16:11,821 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:11,821 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3dbf899733644dbab9eb41aba93a3a74 is 175, key is test_row_0/A:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,822 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into c84caf2fa67b4859b729322269293ce5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:11,822 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:11,822 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=13, startTime=1732317371370; duration=0sec 2024-11-22T23:16:11,823 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:11,823 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:11,823 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:11,823 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:11,824 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:11,824 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:11,824 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/87cd15acbbad43229d42ff0e0b3aeab4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=36.1 K 2024-11-22T23:16:11,825 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 87cd15acbbad43229d42ff0e0b3aeab4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732317366542 2024-11-22T23:16:11,825 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9415cda32360424e969d7a06490bd25b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732317367791 2024-11-22T23:16:11,826 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d10ce1947f948b383dbef30cc32d131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370007 2024-11-22T23:16:11,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742233_1409 (size=31685) 2024-11-22T23:16:11,846 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:11,846 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/0e675db0c6f24ab9bd6b01cfb1d40c30 is 50, key is test_row_0/C:col10/1732317371178/Put/seqid=0 2024-11-22T23:16:11,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742234_1410 (size=12731) 2024-11-22T23:16:11,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317431928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317431928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317431928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317431928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317431937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:11,964 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:11,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e5d60ab5a5584ab59f73a87db190674b_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d60ab5a5584ab59f73a87db190674b_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:11,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/9e4f483fbe244100a702f61e46585a18, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:11,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/9e4f483fbe244100a702f61e46585a18 is 175, key is test_row_0/A:col10/1732317371215/Put/seqid=0 2024-11-22T23:16:11,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742235_1411 (size=39749) 2024-11-22T23:16:11,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/9e4f483fbe244100a702f61e46585a18 2024-11-22T23:16:11,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/59bcd91a8e504fada5e4cd84d0b61de6 is 50, key is test_row_0/B:col10/1732317371215/Put/seqid=0 2024-11-22T23:16:11,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742236_1412 (size=12151) 2024-11-22T23:16:12,246 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/3dbf899733644dbab9eb41aba93a3a74 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74 2024-11-22T23:16:12,250 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into 3dbf899733644dbab9eb41aba93a3a74(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:12,250 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:12,250 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=13, startTime=1732317371370; duration=0sec 2024-11-22T23:16:12,251 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:12,251 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:12,255 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/0e675db0c6f24ab9bd6b01cfb1d40c30 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/0e675db0c6f24ab9bd6b01cfb1d40c30 2024-11-22T23:16:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317432248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317432248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317432248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317432248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317432248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,262 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into 0e675db0c6f24ab9bd6b01cfb1d40c30(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:12,262 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:12,262 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=13, startTime=1732317371370; duration=0sec 2024-11-22T23:16:12,262 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:12,262 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:12,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/59bcd91a8e504fada5e4cd84d0b61de6 2024-11-22T23:16:12,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/8c667277ebcc4c05be18ad7f519bfbea is 50, key is test_row_0/C:col10/1732317371215/Put/seqid=0 2024-11-22T23:16:12,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742237_1413 (size=12151) 2024-11-22T23:16:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-22T23:16:12,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:12,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-22T23:16:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:12,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:12,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:12,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:12,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T23:16:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:12,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:12,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317432767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317432767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317432767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317432767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317432767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:12,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:12,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/8c667277ebcc4c05be18ad7f519bfbea 2024-11-22T23:16:12,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T23:16:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:12,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/9e4f483fbe244100a702f61e46585a18 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18 2024-11-22T23:16:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:12,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18, entries=200, sequenceid=242, filesize=38.8 K 2024-11-22T23:16:12,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/59bcd91a8e504fada5e4cd84d0b61de6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6 2024-11-22T23:16:12,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6, entries=150, sequenceid=242, filesize=11.9 K 2024-11-22T23:16:12,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/8c667277ebcc4c05be18ad7f519bfbea as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea 2024-11-22T23:16:12,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea, entries=150, sequenceid=242, filesize=11.9 K 2024-11-22T23:16:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:12,807 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e92d0941565ecb9e195f60897027f996 in 1253ms, sequenceid=242, compaction requested=false 2024-11-22T23:16:12,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:12,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:12,948 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:12,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112257c971cc92b34a889df2d941d35b5487_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317371602/Put/seqid=0 2024-11-22T23:16:12,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742238_1414 (size=12404) 2024-11-22T23:16:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:13,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:13,363 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112257c971cc92b34a889df2d941d35b5487_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257c971cc92b34a889df2d941d35b5487_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/708570d89ddc49f9bd5b0b3268c9eff3, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/708570d89ddc49f9bd5b0b3268c9eff3 is 175, key is test_row_0/A:col10/1732317371602/Put/seqid=0 2024-11-22T23:16:13,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742239_1415 (size=31205) 2024-11-22T23:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:13,769 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/708570d89ddc49f9bd5b0b3268c9eff3 2024-11-22T23:16:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/efc8efd51512424ca99586c555067bba is 50, key is test_row_0/B:col10/1732317371602/Put/seqid=0 2024-11-22T23:16:13,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742240_1416 (size=12251) 2024-11-22T23:16:13,780 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/efc8efd51512424ca99586c555067bba 2024-11-22T23:16:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:13,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/01b77437133144c2a277544144bec086 is 50, key is test_row_0/C:col10/1732317371602/Put/seqid=0 2024-11-22T23:16:13,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742241_1417 (size=12251) 2024-11-22T23:16:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317433793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317433794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317433797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317433797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317433799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317433907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317433908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317433908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317433917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317433917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317434118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317434118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317434127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317434137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317434137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,190 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/01b77437133144c2a277544144bec086 2024-11-22T23:16:14,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/708570d89ddc49f9bd5b0b3268c9eff3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3 2024-11-22T23:16:14,195 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3, entries=150, sequenceid=265, filesize=30.5 K 2024-11-22T23:16:14,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/efc8efd51512424ca99586c555067bba as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba 2024-11-22T23:16:14,199 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba, entries=150, sequenceid=265, filesize=12.0 K 2024-11-22T23:16:14,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/01b77437133144c2a277544144bec086 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086 2024-11-22T23:16:14,202 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086, entries=150, sequenceid=265, filesize=12.0 K 2024-11-22T23:16:14,203 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e92d0941565ecb9e195f60897027f996 in 1255ms, sequenceid=265, compaction requested=true 2024-11-22T23:16:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-22T23:16:14,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-22T23:16:14,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-22T23:16:14,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-11-22T23:16:14,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7180 sec 2024-11-22T23:16:14,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:14,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122515ae8df88b04c998f0ff3e90e6cbdef_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:14,451 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc] to archive 2024-11-22T23:16:14,452 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:14,453 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/a933247410b24c2db682ff9eb8e74498 2024-11-22T23:16:14,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742242_1418 (size=14994) 2024-11-22T23:16:14,454 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:14,456 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/fcb4082b998742a0b288fa9bb06d526b 2024-11-22T23:16:14,457 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/01415962a30c41049aeb93109cdfa9f8 2024-11-22T23:16:14,458 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/1612b59603ed421b99a2b24d902e4bef 2024-11-22T23:16:14,459 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122515ae8df88b04c998f0ff3e90e6cbdef_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122515ae8df88b04c998f0ff3e90e6cbdef_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:14,459 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3387662f5f794f01a89c8aa6e271c690 2024-11-22T23:16:14,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6027df8d7c9d4bb699e58946e4be3898, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:14,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6027df8d7c9d4bb699e58946e4be3898 is 175, key is test_row_0/A:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:14,460 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/7ac97bac05414e32a3184afe63979f89 2024-11-22T23:16:14,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6cc6aa0d25d84d89a61121ca2c842617 2024-11-22T23:16:14,461 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/035ab2c29c6c4094b5a6f74f8d09c960 2024-11-22T23:16:14,462 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c8c9b628e78743f3a841443f62f1e41a 2024-11-22T23:16:14,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742243_1419 (size=39949) 2024-11-22T23:16:14,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6027df8d7c9d4bb699e58946e4be3898 2024-11-22T23:16:14,463 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/07d7089997c347ee9f783d7a4aa802cd 2024-11-22T23:16:14,464 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/087fb847fbf54f07aa612b6a08a77437 2024-11-22T23:16:14,465 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/477c1156f638422db6a9c354fe0c6564 2024-11-22T23:16:14,466 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/66482da0ef344e679e0bd3eb1c6986dc 2024-11-22T23:16:14,467 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/5068a07763cf43c9b9e06879e7616736 2024-11-22T23:16:14,468 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c78fc86c486a4068adf436615379e5dd 2024-11-22T23:16:14,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/47e2acb03e1841b8874700fb9268096b is 50, key is test_row_0/B:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:14,471 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/8cacffc9828a413ba7c9a99cdc43b282 2024-11-22T23:16:14,472 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/dcf67f2978a4429488bef7c9afe416fc 2024-11-22T23:16:14,475 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b66c6a7b04ff43259f91ccdaf68b9c28, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0029f6e8b1e849bbb375d88bf00d79f7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/17adb30000ae41368e8c5ff9ca72645c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b21878e13ea3458fb4cd2980946ea69b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5] to archive 2024-11-22T23:16:14,476 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:14,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742244_1420 (size=12301) 2024-11-22T23:16:14,478 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2898c520585c4f54a8d178fb2a8fe0cf 2024-11-22T23:16:14,479 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c0d1420c06a2427ca3e3e7ffa2a83920 2024-11-22T23:16:14,480 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b66c6a7b04ff43259f91ccdaf68b9c28 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b66c6a7b04ff43259f91ccdaf68b9c28 2024-11-22T23:16:14,481 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/2437cf25ddfe40dd86e8ffbd10fe5bcf 2024-11-22T23:16:14,482 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/8c84bd27fdea4e8faad0c494edca2ff1 2024-11-22T23:16:14,483 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/f634dda22ac04e8e985c9fee6b988ad1 2024-11-22T23:16:14,487 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0029f6e8b1e849bbb375d88bf00d79f7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0029f6e8b1e849bbb375d88bf00d79f7 2024-11-22T23:16:14,488 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/0d60a75bba124eb59e10c4a98dd8b7ba 2024-11-22T23:16:14,489 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/858afd1169c8463eaa76f51f3a2e11e7 2024-11-22T23:16:14,490 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/ef5a1d8bc3e940219a310b6a3e2d5d7d 2024-11-22T23:16:14,491 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/17adb30000ae41368e8c5ff9ca72645c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/17adb30000ae41368e8c5ff9ca72645c 2024-11-22T23:16:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317434476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317434481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,498 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3e7e0b7fec874e69b3da66c510e487c0 2024-11-22T23:16:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317434486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/6b10a0bd64a4490daf4dfb1ee2ebe863 2024-11-22T23:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b21878e13ea3458fb4cd2980946ea69b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/b21878e13ea3458fb4cd2980946ea69b 2024-11-22T23:16:14,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/aa3ef8aebaad48d3afbe32aa1efff846 2024-11-22T23:16:14,504 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/a2f9a69a3507444996d4d6267271edcd 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317434487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317434492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/d584750119ac4568aec440a29b7e52c5 2024-11-22T23:16:14,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3a725808e40f4c37a9b7c98c35e94e96, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f8112af0bee04fe4ae0cf996bea50781, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/76d7ac3b05d34d9f85cd3c306b43d817, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/87cd15acbbad43229d42ff0e0b3aeab4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131] to archive 2024-11-22T23:16:14,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:14,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/436cb6a240f942f1a79aaba3359cc779 2024-11-22T23:16:14,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c3929eba80a74b9cb9ac6e037093f8b3 2024-11-22T23:16:14,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3a725808e40f4c37a9b7c98c35e94e96 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3a725808e40f4c37a9b7c98c35e94e96 2024-11-22T23:16:14,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/c2c0ee5acd4b478ea2458f6e884cd973 2024-11-22T23:16:14,527 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/d8977d94e73c4087b937b2b213af31ef 2024-11-22T23:16:14,528 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/44b68562929748b192555ddccf51da59 2024-11-22T23:16:14,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f8112af0bee04fe4ae0cf996bea50781 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f8112af0bee04fe4ae0cf996bea50781 2024-11-22T23:16:14,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/83242290f400440caec7edc5dbd6a3f6 2024-11-22T23:16:14,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/5598a65b5e044b5095ba650a4f2a2627 2024-11-22T23:16:14,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/f1207ea3870447a1bba9781c86da6dec 2024-11-22T23:16:14,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/76d7ac3b05d34d9f85cd3c306b43d817 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/76d7ac3b05d34d9f85cd3c306b43d817 2024-11-22T23:16:14,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/a077994232d24bed9b3dee1d44a068e4 2024-11-22T23:16:14,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/929abba400fd423bb90de0759b99946b 2024-11-22T23:16:14,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/87cd15acbbad43229d42ff0e0b3aeab4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/87cd15acbbad43229d42ff0e0b3aeab4 2024-11-22T23:16:14,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b21dafc002fe468c965d15853e4d7f4a 2024-11-22T23:16:14,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/9415cda32360424e969d7a06490bd25b 2024-11-22T23:16:14,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/1d10ce1947f948b383dbef30cc32d131 2024-11-22T23:16:14,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317434608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317434608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317434608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317434608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317434617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-22T23:16:14,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:14,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:14,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-22T23:16:14,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:14,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:14,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:14,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:14,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T23:16:14,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:14,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317434817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317434817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317434817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317434817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317434828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:14,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/47e2acb03e1841b8874700fb9268096b 2024-11-22T23:16:14,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/69ecc012f3244d278912b3fb1a856e49 is 50, key is test_row_0/C:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:14,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742245_1421 (size=12301) 2024-11-22T23:16:14,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T23:16:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:14,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:14,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:15,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T23:16:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317435137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317435137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317435137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,145 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317435137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317435148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T23:16:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:15,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/69ecc012f3244d278912b3fb1a856e49 2024-11-22T23:16:15,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/6027df8d7c9d4bb699e58946e4be3898 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898 2024-11-22T23:16:15,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898, entries=200, sequenceid=278, filesize=39.0 K 2024-11-22T23:16:15,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/47e2acb03e1841b8874700fb9268096b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b 2024-11-22T23:16:15,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-22T23:16:15,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/69ecc012f3244d278912b3fb1a856e49 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49 2024-11-22T23:16:15,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49, entries=150, sequenceid=278, filesize=12.0 K 2024-11-22T23:16:15,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e92d0941565ecb9e195f60897027f996 in 859ms, sequenceid=278, compaction requested=true 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:15,302 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:15,302 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:15,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142588 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49434 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/B is initiating minor compaction (all files) 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/A is initiating minor compaction (all files) 2024-11-22T23:16:15,303 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/B in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,303 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/A in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,303 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=139.2 K 2024-11-22T23:16:15,303 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c84caf2fa67b4859b729322269293ce5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=48.3 K 2024-11-22T23:16:15,303 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898] 2024-11-22T23:16:15,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting c84caf2fa67b4859b729322269293ce5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370007 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dbf899733644dbab9eb41aba93a3a74, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370007 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bcd91a8e504fada5e4cd84d0b61de6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732317371180 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4f483fbe244100a702f61e46585a18, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732317371180 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting efc8efd51512424ca99586c555067bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732317371585 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 708570d89ddc49f9bd5b0b3268c9eff3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732317371585 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 47e2acb03e1841b8874700fb9268096b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732317373780 2024-11-22T23:16:15,304 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6027df8d7c9d4bb699e58946e4be3898, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732317373780 2024-11-22T23:16:15,313 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:15,314 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:15,314 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/077cf7f27cf44a52bbeccbc9243cbdae is 50, key is test_row_0/B:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:15,326 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122d138e9dd8bbb44999acce037c2393c93_e92d0941565ecb9e195f60897027f996 store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:15,328 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122d138e9dd8bbb44999acce037c2393c93_e92d0941565ecb9e195f60897027f996, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:15,328 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122d138e9dd8bbb44999acce037c2393c93_e92d0941565ecb9e195f60897027f996 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742246_1422 (size=12439) 2024-11-22T23:16:15,347 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/077cf7f27cf44a52bbeccbc9243cbdae as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/077cf7f27cf44a52bbeccbc9243cbdae 2024-11-22T23:16:15,352 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/B of e92d0941565ecb9e195f60897027f996 into 077cf7f27cf44a52bbeccbc9243cbdae(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:15,352 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:15,352 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/B, priority=12, startTime=1732317375302; duration=0sec 2024-11-22T23:16:15,352 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:15,352 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:15,352 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:15,353 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49434 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:15,353 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): e92d0941565ecb9e195f60897027f996/C is initiating minor compaction (all files) 2024-11-22T23:16:15,353 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e92d0941565ecb9e195f60897027f996/C in TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,353 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/0e675db0c6f24ab9bd6b01cfb1d40c30, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp, totalSize=48.3 K 2024-11-22T23:16:15,353 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e675db0c6f24ab9bd6b01cfb1d40c30, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732317370007 2024-11-22T23:16:15,353 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c667277ebcc4c05be18ad7f519bfbea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732317371180 2024-11-22T23:16:15,354 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 01b77437133144c2a277544144bec086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732317371585 2024-11-22T23:16:15,354 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ecc012f3244d278912b3fb1a856e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732317373780 2024-11-22T23:16:15,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742247_1423 (size=4469) 2024-11-22T23:16:15,366 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:15,366 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3ea9d952d93f48c386add6eea07dd744 is 50, key is test_row_0/C:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:15,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742248_1424 (size=12439) 2024-11-22T23:16:15,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,414 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:15,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222ce8e11149ce4a689ccf121f03c74800_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317374474/Put/seqid=0 2024-11-22T23:16:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742249_1425 (size=12454) 2024-11-22T23:16:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:15,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222ce8e11149ce4a689ccf121f03c74800_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ce8e11149ce4a689ccf121f03c74800_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:15,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/ae3c463348bd430691376b5858b46911, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/ae3c463348bd430691376b5858b46911 is 175, key is test_row_0/A:col10/1732317374474/Put/seqid=0 2024-11-22T23:16:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742250_1426 (size=31255) 2024-11-22T23:16:15,459 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/ae3c463348bd430691376b5858b46911 2024-11-22T23:16:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3f65b4270e61433e91c66ff07e157c39 is 50, key is test_row_0/B:col10/1732317374474/Put/seqid=0 2024-11-22T23:16:15,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742251_1427 (size=12301) 2024-11-22T23:16:15,504 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3f65b4270e61433e91c66ff07e157c39 2024-11-22T23:16:15,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/2a838b093eeb415e9be909816a013c88 is 50, key is test_row_0/C:col10/1732317374474/Put/seqid=0 2024-11-22T23:16:15,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742252_1428 (size=12301) 2024-11-22T23:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:15,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. as already flushing 2024-11-22T23:16:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41502 deadline: 1732317435668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317435674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317435675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317435677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317435678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,688 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-22T23:16:15,688 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,688 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,688 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,692 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-22T23:16:15,692 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,692 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,692 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,693 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-22T23:16:15,693 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,693 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,693 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,702 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-22T23:16:15,702 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,702 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,702 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,702 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,703 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-22T23:16:15,703 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,703 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,703 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,703 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,763 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e92d0941565ecb9e195f60897027f996#A#compaction#360 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:15,763 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c95e11ac913e4537b1608e6fea0f01e3 is 175, key is test_row_0/A:col10/1732317374442/Put/seqid=0 2024-11-22T23:16:15,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742253_1429 (size=31393) 2024-11-22T23:16:15,779 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3ea9d952d93f48c386add6eea07dd744 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ea9d952d93f48c386add6eea07dd744 2024-11-22T23:16:15,786 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/C of e92d0941565ecb9e195f60897027f996 into 3ea9d952d93f48c386add6eea07dd744(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:15,786 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:15,786 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/C, priority=12, startTime=1732317375302; duration=0sec 2024-11-22T23:16:15,786 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:15,786 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41492 deadline: 1732317435788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41516 deadline: 1732317435789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41518 deadline: 1732317435798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41452 deadline: 1732317435798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:15,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:15,943 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/2a838b093eeb415e9be909816a013c88 2024-11-22T23:16:15,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/ae3c463348bd430691376b5858b46911 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/ae3c463348bd430691376b5858b46911 2024-11-22T23:16:15,957 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/ae3c463348bd430691376b5858b46911, entries=150, sequenceid=298, filesize=30.5 K 2024-11-22T23:16:15,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/3f65b4270e61433e91c66ff07e157c39 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3f65b4270e61433e91c66ff07e157c39 2024-11-22T23:16:15,962 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3f65b4270e61433e91c66ff07e157c39, entries=150, sequenceid=298, filesize=12.0 K 2024-11-22T23:16:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/2a838b093eeb415e9be909816a013c88 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/2a838b093eeb415e9be909816a013c88 2024-11-22T23:16:15,966 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/2a838b093eeb415e9be909816a013c88, entries=150, sequenceid=298, filesize=12.0 K 2024-11-22T23:16:15,967 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for e92d0941565ecb9e195f60897027f996 in 553ms, sequenceid=298, compaction requested=false 2024-11-22T23:16:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:15,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-22T23:16:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-22T23:16:15,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-22T23:16:15,970 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3170 sec 2024-11-22T23:16:15,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3220 sec 2024-11-22T23:16:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:15,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:16:15,999 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-22T23:16:15,999 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,999 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:15,999 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:15,999 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:15,999 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,999 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:15,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:15,999 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:15,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:16,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fc8d48f2d2c347089a9de1fe85758180_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_0/A:col10/1732317375677/Put/seqid=0 2024-11-22T23:16:16,010 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-22T23:16:16,010 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:16,010 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,010 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-22T23:16:16,010 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:16,010 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,010 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742254_1430 (size=12454) 2024-11-22T23:16:16,010 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:16,180 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/c95e11ac913e4537b1608e6fea0f01e3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c95e11ac913e4537b1608e6fea0f01e3 2024-11-22T23:16:16,185 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e92d0941565ecb9e195f60897027f996/A of e92d0941565ecb9e195f60897027f996 into c95e11ac913e4537b1608e6fea0f01e3(size=30.7 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:16,185 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:16,185 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., storeName=e92d0941565ecb9e195f60897027f996/A, priority=12, startTime=1732317375302; duration=0sec 2024-11-22T23:16:16,186 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:16,186 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:16,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:16,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122fc8d48f2d2c347089a9de1fe85758180_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fc8d48f2d2c347089a9de1fe85758180_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:16,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/2cfa2436c68e4f50893f7d31114132b9, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:16,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/2cfa2436c68e4f50893f7d31114132b9 is 175, key is test_row_0/A:col10/1732317375677/Put/seqid=0 2024-11-22T23:16:16,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742255_1431 (size=31255) 2024-11-22T23:16:16,699 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-22T23:16:16,699 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:16,699 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,700 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-22T23:16:16,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:16,798 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3156 rows 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3167 rows 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3153 rows 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1053 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3154 rows 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-11-22T23:16:16,799 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3134 rows 2024-11-22T23:16:16,799 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:16:16,799 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:16:16,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:16,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:16,800 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:16:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:16,803 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317376802"}]},"ts":"1732317376802"} 2024-11-22T23:16:16,804 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:16:16,804 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:16:16,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:16:16,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, UNASSIGN}] 2024-11-22T23:16:16,806 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, UNASSIGN 2024-11-22T23:16:16,806 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:16,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, UNASSIGN because future has completed 2024-11-22T23:16:16,808 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:16:16,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:16,821 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/2cfa2436c68e4f50893f7d31114132b9 2024-11-22T23:16:16,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/059f6e244c404cd8bf4de4fd85819688 is 50, key is test_row_0/B:col10/1732317375677/Put/seqid=0 2024-11-22T23:16:16,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742256_1432 (size=12301) 2024-11-22T23:16:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:16,960 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:16,960 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:16:16,960 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing e92d0941565ecb9e195f60897027f996, disabling compactions & flushes 2024-11-22T23:16:16,960 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:17,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/059f6e244c404cd8bf4de4fd85819688 2024-11-22T23:16:17,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b00d7c0d0c404db2924fb0ac6679c92e is 50, key is test_row_0/C:col10/1732317375677/Put/seqid=0 2024-11-22T23:16:17,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742257_1433 (size=12301) 2024-11-22T23:16:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:17,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b00d7c0d0c404db2924fb0ac6679c92e 2024-11-22T23:16:17,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/2cfa2436c68e4f50893f7d31114132b9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/2cfa2436c68e4f50893f7d31114132b9 2024-11-22T23:16:17,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/2cfa2436c68e4f50893f7d31114132b9, entries=150, sequenceid=314, filesize=30.5 K 2024-11-22T23:16:17,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/059f6e244c404cd8bf4de4fd85819688 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/059f6e244c404cd8bf4de4fd85819688 2024-11-22T23:16:17,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/059f6e244c404cd8bf4de4fd85819688, entries=150, sequenceid=314, filesize=12.0 K 2024-11-22T23:16:17,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/b00d7c0d0c404db2924fb0ac6679c92e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b00d7c0d0c404db2924fb0ac6679c92e 2024-11-22T23:16:17,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b00d7c0d0c404db2924fb0ac6679c92e, entries=150, sequenceid=314, filesize=12.0 K 2024-11-22T23:16:17,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for e92d0941565ecb9e195f60897027f996 in 1677ms, sequenceid=314, compaction requested=true 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e92d0941565ecb9e195f60897027f996: 2024-11-22T23:16:17,677 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. because compaction request was cancelled 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:A 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. after waiting 0 ms 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. because compaction request was cancelled 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e92d0941565ecb9e195f60897027f996:C, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:B 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:17,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. because compaction request was cancelled 2024-11-22T23:16:17,677 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e92d0941565ecb9e195f60897027f996:C 2024-11-22T23:16:17,677 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing e92d0941565ecb9e195f60897027f996 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=A 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=B 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e92d0941565ecb9e195f60897027f996, store=C 2024-11-22T23:16:17,677 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:17,684 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a7b268d9bb8841978880d44148fe97dd_e92d0941565ecb9e195f60897027f996 is 50, key is test_row_1/A:col10/1732317376008/Put/seqid=0 2024-11-22T23:16:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742258_1434 (size=9914) 2024-11-22T23:16:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:18,088 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:18,093 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122a7b268d9bb8841978880d44148fe97dd_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7b268d9bb8841978880d44148fe97dd_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:18,094 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/17edc9255ffc4083a4fc8b7986289f49, store: [table=TestAcidGuarantees family=A region=e92d0941565ecb9e195f60897027f996] 2024-11-22T23:16:18,095 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/17edc9255ffc4083a4fc8b7986289f49 is 175, key is test_row_1/A:col10/1732317376008/Put/seqid=0 2024-11-22T23:16:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742259_1435 (size=22561) 2024-11-22T23:16:18,500 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/17edc9255ffc4083a4fc8b7986289f49 2024-11-22T23:16:18,507 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/26a9ecfe33874f04b488deee679abd37 is 50, key is test_row_1/B:col10/1732317376008/Put/seqid=0 2024-11-22T23:16:18,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742260_1436 (size=9857) 2024-11-22T23:16:18,911 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/26a9ecfe33874f04b488deee679abd37 2024-11-22T23:16:18,917 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3ecc0a160488458da78eca2305086851 is 50, key is test_row_1/C:col10/1732317376008/Put/seqid=0 2024-11-22T23:16:18,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742261_1437 (size=9857) 2024-11-22T23:16:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:19,320 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3ecc0a160488458da78eca2305086851 2024-11-22T23:16:19,324 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/A/17edc9255ffc4083a4fc8b7986289f49 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/17edc9255ffc4083a4fc8b7986289f49 2024-11-22T23:16:19,328 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/17edc9255ffc4083a4fc8b7986289f49, entries=100, sequenceid=321, filesize=22.0 K 2024-11-22T23:16:19,328 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/B/26a9ecfe33874f04b488deee679abd37 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/26a9ecfe33874f04b488deee679abd37 2024-11-22T23:16:19,331 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/26a9ecfe33874f04b488deee679abd37, entries=100, sequenceid=321, filesize=9.6 K 2024-11-22T23:16:19,331 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/.tmp/C/3ecc0a160488458da78eca2305086851 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ecc0a160488458da78eca2305086851 2024-11-22T23:16:19,334 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ecc0a160488458da78eca2305086851, entries=100, sequenceid=321, filesize=9.6 K 2024-11-22T23:16:19,334 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e92d0941565ecb9e195f60897027f996 in 1657ms, sequenceid=321, compaction requested=true 2024-11-22T23:16:19,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898] to archive 2024-11-22T23:16:19,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:19,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/3dbf899733644dbab9eb41aba93a3a74 2024-11-22T23:16:19,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/9e4f483fbe244100a702f61e46585a18 2024-11-22T23:16:19,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/708570d89ddc49f9bd5b0b3268c9eff3 2024-11-22T23:16:19,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/6027df8d7c9d4bb699e58946e4be3898 2024-11-22T23:16:19,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c84caf2fa67b4859b729322269293ce5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b] to archive 2024-11-22T23:16:19,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:19,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c84caf2fa67b4859b729322269293ce5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/c84caf2fa67b4859b729322269293ce5 2024-11-22T23:16:19,342 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/59bcd91a8e504fada5e4cd84d0b61de6 2024-11-22T23:16:19,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/efc8efd51512424ca99586c555067bba 2024-11-22T23:16:19,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/47e2acb03e1841b8874700fb9268096b 2024-11-22T23:16:19,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/0e675db0c6f24ab9bd6b01cfb1d40c30, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49] to archive 2024-11-22T23:16:19,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:19,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/0e675db0c6f24ab9bd6b01cfb1d40c30 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/0e675db0c6f24ab9bd6b01cfb1d40c30 2024-11-22T23:16:19,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/8c667277ebcc4c05be18ad7f519bfbea 2024-11-22T23:16:19,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/01b77437133144c2a277544144bec086 2024-11-22T23:16:19,348 DEBUG [StoreCloser-TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/69ecc012f3244d278912b3fb1a856e49 2024-11-22T23:16:19,352 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-22T23:16:19,352 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. 2024-11-22T23:16:19,352 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for e92d0941565ecb9e195f60897027f996: Waiting for close lock at 1732317376960Running coprocessor pre-close hooks at 1732317376960Disabling compacts and flushes for region at 1732317376960Disabling writes for close at 1732317377677 (+717 ms)Obtaining lock to block concurrent updates at 1732317377677Preparing flush snapshotting stores in e92d0941565ecb9e195f60897027f996 at 1732317377677Finished memstore snapshotting TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732317377678 (+1 ms)Flushing stores of TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996. at 1732317377678Flushing e92d0941565ecb9e195f60897027f996/A: creating writer at 1732317377678Flushing mob file e92d0941565ecb9e195f60897027f996/A: appending metadata at 1732317377683 (+5 ms)Flushing mob file e92d0941565ecb9e195f60897027f996/A: closing flushed file at 1732317377683Flushing e92d0941565ecb9e195f60897027f996/A: appending metadata at 1732317378094 (+411 ms)Flushing e92d0941565ecb9e195f60897027f996/A: closing flushed file at 1732317378094Flushing e92d0941565ecb9e195f60897027f996/B: creating writer at 1732317378504 (+410 ms)Flushing e92d0941565ecb9e195f60897027f996/B: appending metadata at 1732317378506 (+2 ms)Flushing e92d0941565ecb9e195f60897027f996/B: closing flushed file at 1732317378506Flushing e92d0941565ecb9e195f60897027f996/C: creating writer at 1732317378914 (+408 ms)Flushing e92d0941565ecb9e195f60897027f996/C: appending metadata at 1732317378916 (+2 ms)Flushing e92d0941565ecb9e195f60897027f996/C: closing flushed file at 1732317378916Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26eb96c7: reopening flushed file at 1732317379324 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@99b9e0: reopening flushed file at 1732317379328 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dac9fbf: reopening flushed file at 1732317379331 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e92d0941565ecb9e195f60897027f996 in 1657ms, sequenceid=321, compaction requested=true at 1732317379334 (+3 ms)Writing region close event to WAL at 1732317379349 (+15 ms)Running coprocessor post-close hooks at 1732317379352 (+3 ms)Closed at 1732317379352 2024-11-22T23:16:19,354 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:19,354 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e92d0941565ecb9e195f60897027f996, regionState=CLOSED 2024-11-22T23:16:19,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:19,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-22T23:16:19,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure e92d0941565ecb9e195f60897027f996, server=a3d87a7361f3,46715,1732317251834 in 2.5480 sec 2024-11-22T23:16:19,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-22T23:16:19,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e92d0941565ecb9e195f60897027f996, UNASSIGN in 2.5530 sec 2024-11-22T23:16:19,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-22T23:16:19,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5560 sec 2024-11-22T23:16:19,362 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317379362"}]},"ts":"1732317379362"} 2024-11-22T23:16:19,363 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:16:19,363 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:16:19,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5630 sec 2024-11-22T23:16:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-22T23:16:20,959 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:20,960 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:16:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:20,964 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:16:20,966 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:20,971 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:20,976 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits] 2024-11-22T23:16:20,981 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/17edc9255ffc4083a4fc8b7986289f49 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/17edc9255ffc4083a4fc8b7986289f49 2024-11-22T23:16:20,983 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/2cfa2436c68e4f50893f7d31114132b9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/2cfa2436c68e4f50893f7d31114132b9 2024-11-22T23:16:20,986 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/ae3c463348bd430691376b5858b46911 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/ae3c463348bd430691376b5858b46911 2024-11-22T23:16:20,988 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c95e11ac913e4537b1608e6fea0f01e3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/A/c95e11ac913e4537b1608e6fea0f01e3 2024-11-22T23:16:20,991 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/059f6e244c404cd8bf4de4fd85819688 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/059f6e244c404cd8bf4de4fd85819688 2024-11-22T23:16:20,992 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/077cf7f27cf44a52bbeccbc9243cbdae to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/077cf7f27cf44a52bbeccbc9243cbdae 2024-11-22T23:16:20,993 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/26a9ecfe33874f04b488deee679abd37 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/26a9ecfe33874f04b488deee679abd37 2024-11-22T23:16:20,994 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3f65b4270e61433e91c66ff07e157c39 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/B/3f65b4270e61433e91c66ff07e157c39 2024-11-22T23:16:20,996 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/2a838b093eeb415e9be909816a013c88 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/2a838b093eeb415e9be909816a013c88 2024-11-22T23:16:20,997 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ea9d952d93f48c386add6eea07dd744 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ea9d952d93f48c386add6eea07dd744 2024-11-22T23:16:20,998 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ecc0a160488458da78eca2305086851 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/3ecc0a160488458da78eca2305086851 2024-11-22T23:16:20,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b00d7c0d0c404db2924fb0ac6679c92e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/C/b00d7c0d0c404db2924fb0ac6679c92e 2024-11-22T23:16:21,001 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits/324.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996/recovered.edits/324.seqid 2024-11-22T23:16:21,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,002 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:16:21,002 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:16:21,003 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:16:21,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220475b70740a54a7aaba7b1a03bd99664_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220475b70740a54a7aaba7b1a03bd99664_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,005 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220f1d3e0d643b4dc285d41ea4cf47ec61_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411220f1d3e0d643b4dc285d41ea4cf47ec61_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,006 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ce8e11149ce4a689ccf121f03c74800_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222ce8e11149ce4a689ccf121f03c74800_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232b55210e09f45b4a8a0d500630487c7_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112232b55210e09f45b4a8a0d500630487c7_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ebf53cf68904eecabc6fc0fa110e9c3_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223ebf53cf68904eecabc6fc0fa110e9c3_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122494961775f694d498048d8a114714ac0_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122494961775f694d498048d8a114714ac0_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,009 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224ad0ac2bef0f4cc4915da99514deb757_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411224ad0ac2bef0f4cc4915da99514deb757_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,010 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122515ae8df88b04c998f0ff3e90e6cbdef_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122515ae8df88b04c998f0ff3e90e6cbdef_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,010 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122578b99b5b3734fcd9bad5efc3b3c62bc_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122578b99b5b3734fcd9bad5efc3b3c62bc_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257c971cc92b34a889df2d941d35b5487_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112257c971cc92b34a889df2d941d35b5487_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226824e65d27f84450981a6a1ee8421a8a_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411226824e65d27f84450981a6a1ee8421a8a_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b3b7a3171784af9b19c8aa318f4b961_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227b3b7a3171784af9b19c8aa318f4b961_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dc7a2b5a7d6453b8b2779451b08473f_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411228dc7a2b5a7d6453b8b2779451b08473f_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7b268d9bb8841978880d44148fe97dd_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122a7b268d9bb8841978880d44148fe97dd_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c273d3fbdbff45baaa01999b9e4fd0e9_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122c273d3fbdbff45baaa01999b9e4fd0e9_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,015 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d60ab5a5584ab59f73a87db190674b_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e5d60ab5a5584ab59f73a87db190674b_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e9370b9881074748b89dafdc52b1f829_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e9370b9881074748b89dafdc52b1f829_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f92b6e57c5664526bcb41e7ba5d16d4d_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122f92b6e57c5664526bcb41e7ba5d16d4d_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fc8d48f2d2c347089a9de1fe85758180_e92d0941565ecb9e195f60897027f996 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122fc8d48f2d2c347089a9de1fe85758180_e92d0941565ecb9e195f60897027f996 2024-11-22T23:16:21,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:16:21,018 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:21,020 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:16:21,022 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:16:21,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:21,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:16:21,023 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317381023"}]},"ts":"9223372036854775807"} 2024-11-22T23:16:21,025 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:16:21,025 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e92d0941565ecb9e195f60897027f996, NAME => 'TestAcidGuarantees,,1732317352522.e92d0941565ecb9e195f60897027f996.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:16:21,025 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:16:21,025 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317381025"}]},"ts":"9223372036854775807"} 2024-11-22T23:16:21,026 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:16:21,027 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:21,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-22T23:16:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-22T23:16:21,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:16:21,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:21,074 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:16:21,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=451 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 367) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=655 (was 1001) 2024-11-22T23:16:21,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=654 2024-11-22T23:16:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:16:21,090 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:16:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:21,092 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:16:21,092 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:21,092 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-22T23:16:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:21,093 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:16:21,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742262_1438 (size=1061) 2024-11-22T23:16:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:21,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:16:21,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742263_1439 (size=53) 2024-11-22T23:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8a67cd775c3b32775e1402006c75b11e, disabling compactions & flushes 2024-11-22T23:16:21,915 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. after waiting 0 ms 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:21,915 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:21,915 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a67cd775c3b32775e1402006c75b11e: Waiting for close lock at 1732317381915Disabling compacts and flushes for region at 1732317381915Disabling writes for close at 1732317381915Writing region close event to WAL at 1732317381915Closed at 1732317381915 2024-11-22T23:16:21,918 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:16:21,919 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317381919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317381919"}]},"ts":"1732317381919"} 2024-11-22T23:16:21,923 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:16:21,925 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:16:21,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317381925"}]},"ts":"1732317381925"} 2024-11-22T23:16:21,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:16:21,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, ASSIGN}] 2024-11-22T23:16:21,928 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, ASSIGN 2024-11-22T23:16:21,929 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:16:22,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=8a67cd775c3b32775e1402006c75b11e, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:22,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, ASSIGN because future has completed 2024-11-22T23:16:22,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:22,238 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:22,238 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:16:22,239 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,239 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:16:22,239 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,239 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,240 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,241 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:22,242 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a67cd775c3b32775e1402006c75b11e columnFamilyName A 2024-11-22T23:16:22,242 DEBUG [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:22,242 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(327): Store=8a67cd775c3b32775e1402006c75b11e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:22,242 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,243 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:22,243 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a67cd775c3b32775e1402006c75b11e columnFamilyName B 2024-11-22T23:16:22,243 DEBUG [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:22,243 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(327): Store=8a67cd775c3b32775e1402006c75b11e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:22,243 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,244 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:22,244 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a67cd775c3b32775e1402006c75b11e columnFamilyName C 2024-11-22T23:16:22,244 DEBUG [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:22,245 INFO [StoreOpener-8a67cd775c3b32775e1402006c75b11e-1 {}] regionserver.HStore(327): Store=8a67cd775c3b32775e1402006c75b11e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:22,245 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:22,245 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,245 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,246 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,246 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,246 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,247 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:16:22,248 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,249 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:16:22,250 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened 8a67cd775c3b32775e1402006c75b11e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63641724, jitterRate=-0.05166441202163696}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:16:22,250 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:22,250 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for 8a67cd775c3b32775e1402006c75b11e: Running coprocessor pre-open hook at 1732317382239Writing region info on filesystem at 1732317382239Initializing all the Stores at 1732317382240 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317382240Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317382240Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317382240Cleaning up temporary data from old regions at 1732317382246 (+6 ms)Running coprocessor post-open hooks at 1732317382250 (+4 ms)Region opened successfully at 1732317382250 2024-11-22T23:16:22,251 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., pid=130, masterSystemTime=1732317382236 2024-11-22T23:16:22,252 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:22,253 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:22,253 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=8a67cd775c3b32775e1402006c75b11e, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:22,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:22,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-22T23:16:22,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 in 172 msec 2024-11-22T23:16:22,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-22T23:16:22,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, ASSIGN in 330 msec 2024-11-22T23:16:22,259 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:16:22,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317382259"}]},"ts":"1732317382259"} 2024-11-22T23:16:22,260 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:16:22,261 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:16:22,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-22T23:16:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-22T23:16:23,248 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:23,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,250 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b92c4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c12f4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,253 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,256 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:16:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@681d1e32 2024-11-22T23:16:23,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:16:23,258 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:16:23,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,260 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c86964a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d5022e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f272363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e0ca93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35dd6c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5672ef3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,270 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acd2666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,272 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cba808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,273 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,273 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,274 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a097f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,276 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3080323e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,278 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56858065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,280 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2402074f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,283 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2563d1b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b48acfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,288 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d44ca42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,290 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6385f1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,293 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dee3b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab2482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:23,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:23,295 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:23,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38295398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:23,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:23,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:23,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d5e73fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:23,297 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,297 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,298 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,298 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:23,298 DEBUG [Thread-1980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,298 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,298 DEBUG [Thread-1997 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,299 DEBUG [Thread-1984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,299 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,300 DEBUG [Thread-1982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,301 DEBUG [Thread-1986 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:23,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,301 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,302 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:23,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:23,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:23,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,305 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,306 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:23,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2] 2024-11-22T23:16:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:23,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:23,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:23,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/1b4ea58437164ab4923b9edc7a2eb4d6 is 50, key is test_row_0/A:col10/1732317383306/Put/seqid=0 2024-11-22T23:16:23,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742264_1440 (size=19021) 2024-11-22T23:16:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317443365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317443366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317383326, max timestamp=1732317383380Num uniques 859; 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317383326, max timestamp=1732317383380Num uniques 859; 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317383326, max timestamp=1732317383380Num uniques 859; 2024-11-22T23:16:23,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:23,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317443411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317443419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317443420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317443487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317443487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317443527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317443537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317443537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:23,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317443698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317443707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/1b4ea58437164ab4923b9edc7a2eb4d6 2024-11-22T23:16:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317443737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/223a986d2dec49edad0cb043d439be6c is 50, key is test_row_0/B:col10/1732317383306/Put/seqid=0 2024-11-22T23:16:23,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317443757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317443758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:23,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742265_1441 (size=12001) 2024-11-22T23:16:23,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:23,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:23,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:23,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:24,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317444017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317444027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317444058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317444077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317444077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/223a986d2dec49edad0cb043d439be6c 2024-11-22T23:16:24,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/73703a28962c402ea00d97ea6c672a04 is 50, key is test_row_0/C:col10/1732317383306/Put/seqid=0 2024-11-22T23:16:24,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742266_1442 (size=12001) 2024-11-22T23:16:24,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:24,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:24,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317444538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317444547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317444577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/73703a28962c402ea00d97ea6c672a04 2024-11-22T23:16:24,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/1b4ea58437164ab4923b9edc7a2eb4d6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6 2024-11-22T23:16:24,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6, entries=300, sequenceid=12, filesize=18.6 K 2024-11-22T23:16:24,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/223a986d2dec49edad0cb043d439be6c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c 2024-11-22T23:16:24,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317444587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:16:24,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/73703a28962c402ea00d97ea6c672a04 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04 2024-11-22T23:16:24,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04, entries=150, sequenceid=12, filesize=11.7 K 2024-11-22T23:16:24,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317444598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:24,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:24,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8a67cd775c3b32775e1402006c75b11e in 1291ms, sequenceid=12, compaction requested=false 2024-11-22T23:16:24,603 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-22T23:16:24,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:24,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:24,681 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:24,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:24,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc762ef4082a4a1eac3ab0244714754c is 50, key is test_row_0/A:col10/1732317383419/Put/seqid=0 2024-11-22T23:16:24,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742267_1443 (size=12001) 2024-11-22T23:16:24,690 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc762ef4082a4a1eac3ab0244714754c 2024-11-22T23:16:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/f099c0470eca43409aede39cef7f0184 is 50, key is test_row_0/B:col10/1732317383419/Put/seqid=0 2024-11-22T23:16:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742268_1444 (size=12001) 2024-11-22T23:16:24,700 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/f099c0470eca43409aede39cef7f0184 2024-11-22T23:16:24,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/abd6aa07b4e24fd0b1c1a52740fa6f04 is 50, key is test_row_0/C:col10/1732317383419/Put/seqid=0 2024-11-22T23:16:24,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742269_1445 (size=12001) 2024-11-22T23:16:24,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:16:25,109 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/abd6aa07b4e24fd0b1c1a52740fa6f04 2024-11-22T23:16:25,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc762ef4082a4a1eac3ab0244714754c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c 2024-11-22T23:16:25,116 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c, entries=150, sequenceid=46, filesize=11.7 K 2024-11-22T23:16:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/f099c0470eca43409aede39cef7f0184 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184 2024-11-22T23:16:25,120 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184, entries=150, sequenceid=46, filesize=11.7 K 2024-11-22T23:16:25,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/abd6aa07b4e24fd0b1c1a52740fa6f04 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04 2024-11-22T23:16:25,123 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04, entries=150, sequenceid=46, filesize=11.7 K 2024-11-22T23:16:25,124 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for 8a67cd775c3b32775e1402006c75b11e in 443ms, sequenceid=46, compaction requested=false 2024-11-22T23:16:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-22T23:16:25,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-22T23:16:25,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-22T23:16:25,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-22T23:16:25,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8290 sec 2024-11-22T23:16:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-22T23:16:25,457 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:25,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-22T23:16:25,459 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:25,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:25,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:25,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:25,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:25,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/dc4c57161ce64e0b86b8721a5e6093fc is 50, key is test_row_0/A:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:25,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742270_1446 (size=16681) 2024-11-22T23:16:25,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317385609, max timestamp=1732317385629Num uniques 859; 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317385609, max timestamp=1732317385629Num uniques 859; 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732317385609, max timestamp=1732317385629Num uniques 859; 2024-11-22T23:16:25,632 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-22T23:16:25,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317445657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317445658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317445659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317445661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317445666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:25,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:25,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317445777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317445788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317445788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:25,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:25,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:25,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/dc4c57161ce64e0b86b8721a5e6093fc 2024-11-22T23:16:25,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/a91bdb0ec46e46f9b63af20f922b8fb5 is 50, key is test_row_0/B:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:26,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742271_1447 (size=12001) 2024-11-22T23:16:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317445998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:26,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317446318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317446318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317446318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317446318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317446318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/a91bdb0ec46e46f9b63af20f922b8fb5 2024-11-22T23:16:26,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/821e8604798a49cb9047496d4d875974 is 50, key is test_row_0/C:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:26,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742272_1448 (size=12001) 2024-11-22T23:16:26,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:26,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:26,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/821e8604798a49cb9047496d4d875974 2024-11-22T23:16:26,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/dc4c57161ce64e0b86b8721a5e6093fc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc 2024-11-22T23:16:26,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc, entries=250, sequenceid=56, filesize=16.3 K 2024-11-22T23:16:26,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/a91bdb0ec46e46f9b63af20f922b8fb5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5 2024-11-22T23:16:26,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5, entries=150, sequenceid=56, filesize=11.7 K 2024-11-22T23:16:26,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/821e8604798a49cb9047496d4d875974 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974 2024-11-22T23:16:26,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974, entries=150, sequenceid=56, filesize=11.7 K 2024-11-22T23:16:26,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8a67cd775c3b32775e1402006c75b11e in 1254ms, sequenceid=56, compaction requested=true 2024-11-22T23:16:26,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:26,827 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:26,827 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,827 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,828 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.2 K 2024-11-22T23:16:26,828 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=46.6 K 2024-11-22T23:16:26,828 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b4ea58437164ab4923b9edc7a2eb4d6, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317383306 2024-11-22T23:16:26,828 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc762ef4082a4a1eac3ab0244714754c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732317383409 2024-11-22T23:16:26,830 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 223a986d2dec49edad0cb043d439be6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317383306 2024-11-22T23:16:26,831 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc4c57161ce64e0b86b8721a5e6093fc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:26,831 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f099c0470eca43409aede39cef7f0184, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732317383409 2024-11-22T23:16:26,831 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a91bdb0ec46e46f9b63af20f922b8fb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:26,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-22T23:16:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:26,834 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-22T23:16:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:26,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:26,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:26,839 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:26,839 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/cad8b537ac7b46b2b16a4523cbf9967a is 50, key is test_row_0/A:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:26,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:26,842 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:26,843 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/f5a47f4793dc40d498c5defd62a63edc is 50, key is test_row_0/B:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc45aa5236814796b9d50ea1f08a4eb1 is 50, key is test_row_0/A:col10/1732317385665/Put/seqid=0 2024-11-22T23:16:26,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742274_1450 (size=12104) 2024-11-22T23:16:26,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742273_1449 (size=12104) 2024-11-22T23:16:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317446845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317446846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317446849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742275_1451 (size=12001) 2024-11-22T23:16:26,862 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc45aa5236814796b9d50ea1f08a4eb1 2024-11-22T23:16:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317446854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317446855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bc7a4fe2d42449efbf8f95c953aa32ed is 50, key is test_row_0/B:col10/1732317385665/Put/seqid=0 2024-11-22T23:16:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742276_1452 (size=12001) 2024-11-22T23:16:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317446968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317446977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317447187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,255 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/f5a47f4793dc40d498c5defd62a63edc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f5a47f4793dc40d498c5defd62a63edc 2024-11-22T23:16:27,258 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into f5a47f4793dc40d498c5defd62a63edc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:27,258 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:27,258 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317386827; duration=0sec 2024-11-22T23:16:27,259 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:27,259 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:27,259 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:27,260 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:27,260 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:27,260 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:27,260 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.2 K 2024-11-22T23:16:27,260 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 73703a28962c402ea00d97ea6c672a04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732317383306 2024-11-22T23:16:27,260 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting abd6aa07b4e24fd0b1c1a52740fa6f04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732317383409 2024-11-22T23:16:27,261 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/cad8b537ac7b46b2b16a4523cbf9967a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cad8b537ac7b46b2b16a4523cbf9967a 2024-11-22T23:16:27,261 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 821e8604798a49cb9047496d4d875974, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:27,264 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into cad8b537ac7b46b2b16a4523cbf9967a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:27,264 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:27,264 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317386826; duration=0sec 2024-11-22T23:16:27,264 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:27,264 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:27,266 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:27,267 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/9c812d21e68d4fb4a948dd5d3b41944e is 50, key is test_row_0/C:col10/1732317385569/Put/seqid=0 2024-11-22T23:16:27,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742277_1453 (size=12104) 2024-11-22T23:16:27,270 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bc7a4fe2d42449efbf8f95c953aa32ed 2024-11-22T23:16:27,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/1a912c6378c04d899628c204d2f31ef8 is 50, key is test_row_0/C:col10/1732317385665/Put/seqid=0 2024-11-22T23:16:27,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742278_1454 (size=12001) 2024-11-22T23:16:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317447507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:27,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:27,674 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/9c812d21e68d4fb4a948dd5d3b41944e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/9c812d21e68d4fb4a948dd5d3b41944e 2024-11-22T23:16:27,679 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into 9c812d21e68d4fb4a948dd5d3b41944e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:27,679 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:27,679 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317386827; duration=0sec 2024-11-22T23:16:27,679 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:27,679 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:27,680 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/1a912c6378c04d899628c204d2f31ef8 2024-11-22T23:16:27,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/fc45aa5236814796b9d50ea1f08a4eb1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1 2024-11-22T23:16:27,687 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1, entries=150, sequenceid=90, filesize=11.7 K 2024-11-22T23:16:27,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bc7a4fe2d42449efbf8f95c953aa32ed as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed 2024-11-22T23:16:27,691 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed, entries=150, sequenceid=90, filesize=11.7 K 2024-11-22T23:16:27,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/1a912c6378c04d899628c204d2f31ef8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8 2024-11-22T23:16:27,700 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8, entries=150, sequenceid=90, filesize=11.7 K 2024-11-22T23:16:27,700 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 8a67cd775c3b32775e1402006c75b11e in 866ms, sequenceid=90, compaction requested=false 2024-11-22T23:16:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-22T23:16:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-22T23:16:27,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-22T23:16:27,704 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2420 sec 2024-11-22T23:16:27,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.2490 sec 2024-11-22T23:16:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:28,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:28,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:28,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e2937ebf14d49a4805e34d8d615af46 is 50, key is test_row_0/A:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:28,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742279_1455 (size=9657) 2024-11-22T23:16:28,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317448060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317448060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317448062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317448065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317448067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317448178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317448180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317448187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317448187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317448187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317448387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317448387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317448408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317448408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317448417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e2937ebf14d49a4805e34d8d615af46 2024-11-22T23:16:28,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bbd995632bc840ec87ae4bcd50d0fbcf is 50, key is test_row_0/B:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:28,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742280_1456 (size=9657) 2024-11-22T23:16:28,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317448697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317448707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317448717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317448717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317448728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:28,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:28,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bbd995632bc840ec87ae4bcd50d0fbcf 2024-11-22T23:16:28,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/fa36ed7c57db4c2593eef9bec96b4ae9 is 50, key is test_row_0/C:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:28,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742281_1457 (size=9657) 2024-11-22T23:16:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317449217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:29,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317449217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:29,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317449227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317449238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:29,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317449238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:29,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:29,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/fa36ed7c57db4c2593eef9bec96b4ae9 2024-11-22T23:16:29,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e2937ebf14d49a4805e34d8d615af46 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46 2024-11-22T23:16:29,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46, entries=100, sequenceid=105, filesize=9.4 K 2024-11-22T23:16:29,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/bbd995632bc840ec87ae4bcd50d0fbcf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf 2024-11-22T23:16:29,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf, entries=100, sequenceid=105, filesize=9.4 K 2024-11-22T23:16:29,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/fa36ed7c57db4c2593eef9bec96b4ae9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9 2024-11-22T23:16:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9, entries=100, sequenceid=105, filesize=9.4 K 2024-11-22T23:16:29,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a67cd775c3b32775e1402006c75b11e in 1249ms, sequenceid=105, compaction requested=true 2024-11-22T23:16:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:29,281 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:29,281 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:29,282 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:29,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:29,282 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:29,282 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:29,282 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:29,282 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:29,282 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:29,282 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:29,282 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cad8b537ac7b46b2b16a4523cbf9967a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=33.0 K 2024-11-22T23:16:29,282 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f5a47f4793dc40d498c5defd62a63edc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=33.0 K 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cad8b537ac7b46b2b16a4523cbf9967a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a47f4793dc40d498c5defd62a63edc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc45aa5236814796b9d50ea1f08a4eb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732317385647 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7a4fe2d42449efbf8f95c953aa32ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732317385647 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bbd995632bc840ec87ae4bcd50d0fbcf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317386854 2024-11-22T23:16:29,283 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2937ebf14d49a4805e34d8d615af46, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317386854 2024-11-22T23:16:29,289 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#390 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:29,289 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:29,290 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/62263019653f4a01b0481cbdfbfb222a is 50, key is test_row_0/B:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:29,290 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/bc677982de8943be91a84e1e76d80063 is 50, key is test_row_0/A:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742282_1458 (size=12207) 2024-11-22T23:16:29,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742283_1459 (size=12207) 2024-11-22T23:16:29,298 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/bc677982de8943be91a84e1e76d80063 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/bc677982de8943be91a84e1e76d80063 2024-11-22T23:16:29,299 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/62263019653f4a01b0481cbdfbfb222a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/62263019653f4a01b0481cbdfbfb222a 2024-11-22T23:16:29,302 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into 62263019653f4a01b0481cbdfbfb222a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:29,302 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into bc677982de8943be91a84e1e76d80063(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:29,302 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317389281; duration=0sec 2024-11-22T23:16:29,302 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317389281; duration=0sec 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:29,302 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:29,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:29,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:29,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:29,303 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:29,303 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/9c812d21e68d4fb4a948dd5d3b41944e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=33.0 K 2024-11-22T23:16:29,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c812d21e68d4fb4a948dd5d3b41944e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732317385557 2024-11-22T23:16:29,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a912c6378c04d899628c204d2f31ef8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732317385647 2024-11-22T23:16:29,304 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fa36ed7c57db4c2593eef9bec96b4ae9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317386854 2024-11-22T23:16:29,309 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:29,309 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/6f9283cd36874b7d8ea6d4aa06633fae is 50, key is test_row_0/C:col10/1732317388032/Put/seqid=0 2024-11-22T23:16:29,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742284_1460 (size=12207) 2024-11-22T23:16:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-22T23:16:29,618 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:29,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-22T23:16:29,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:16:29,620 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:29,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:29,716 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/6f9283cd36874b7d8ea6d4aa06633fae as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6f9283cd36874b7d8ea6d4aa06633fae 2024-11-22T23:16:29,720 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into 6f9283cd36874b7d8ea6d4aa06633fae(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:29,720 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:29,720 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317389282; duration=0sec 2024-11-22T23:16:29,720 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:29,720 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:16:29,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:29,772 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:29,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:29,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/46fdfa605d5542d3a5a3316b95633a63 is 50, key is test_row_0/A:col10/1732317388060/Put/seqid=0 2024-11-22T23:16:29,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742285_1461 (size=12001) 2024-11-22T23:16:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:16:30,179 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/46fdfa605d5542d3a5a3316b95633a63 2024-11-22T23:16:30,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ce3b400a90424fc0a93e20fd8dd22e07 is 50, key is test_row_0/B:col10/1732317388060/Put/seqid=0 2024-11-22T23:16:30,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742286_1462 (size=12001) 2024-11-22T23:16:30,195 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ce3b400a90424fc0a93e20fd8dd22e07 2024-11-22T23:16:30,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c2add79512a04024af90963470d199cb is 50, key is test_row_0/C:col10/1732317388060/Put/seqid=0 2024-11-22T23:16:30,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742287_1463 (size=12001) 2024-11-22T23:16:30,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:30,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:16:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317450249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317450252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317450257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317450259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,262 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317450260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317450367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317450367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317450367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317450378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317450577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317450587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317450587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317450598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,606 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c2add79512a04024af90963470d199cb 2024-11-22T23:16:30,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/46fdfa605d5542d3a5a3316b95633a63 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63 2024-11-22T23:16:30,613 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63, entries=150, sequenceid=128, filesize=11.7 K 2024-11-22T23:16:30,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ce3b400a90424fc0a93e20fd8dd22e07 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07 2024-11-22T23:16:30,617 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07, entries=150, sequenceid=128, filesize=11.7 K 2024-11-22T23:16:30,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c2add79512a04024af90963470d199cb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb 2024-11-22T23:16:30,620 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb, entries=150, sequenceid=128, filesize=11.7 K 2024-11-22T23:16:30,621 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a67cd775c3b32775e1402006c75b11e in 849ms, sequenceid=128, compaction requested=false 2024-11-22T23:16:30,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:30,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:30,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-22T23:16:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-22T23:16:30,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-22T23:16:30,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0020 sec 2024-11-22T23:16:30,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0060 sec 2024-11-22T23:16:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-22T23:16:30,757 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:30,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-22T23:16:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:30,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:30,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:30,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:30,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:30,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:30,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:30,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/a1a135fff19d44f3b723466f7a9a4fce is 50, key is test_row_0/A:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:30,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742288_1464 (size=14541) 2024-11-22T23:16:30,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:30,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:30,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:30,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317450929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317450929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317450930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317450931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:30,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:30,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317451047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317451047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317451047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317451047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,074 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-22T23:16:31,074 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-22T23:16:31,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:31,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317451257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317451257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317451257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317451257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/a1a135fff19d44f3b723466f7a9a4fce 2024-11-22T23:16:31,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/13a8155aa8d94c48ad40eaea2c2361c0 is 50, key is test_row_0/B:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:31,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742289_1465 (size=12151) 2024-11-22T23:16:31,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:31,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317451577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317451577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317451577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317451577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:31,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:31,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/13a8155aa8d94c48ad40eaea2c2361c0 2024-11-22T23:16:31,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f3b8d956c75b4406af3bbb159e70e7f5 is 50, key is test_row_0/C:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:31,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742290_1466 (size=12151) 2024-11-22T23:16:31,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:31,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:31,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:31,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:31,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:32,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317452088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317452088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317452097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317452097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f3b8d956c75b4406af3bbb159e70e7f5 2024-11-22T23:16:32,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/a1a135fff19d44f3b723466f7a9a4fce as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce 2024-11-22T23:16:32,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce, entries=200, sequenceid=141, filesize=14.2 K 2024-11-22T23:16:32,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/13a8155aa8d94c48ad40eaea2c2361c0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0 2024-11-22T23:16:32,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0, entries=150, sequenceid=141, filesize=11.9 K 2024-11-22T23:16:32,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f3b8d956c75b4406af3bbb159e70e7f5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5 2024-11-22T23:16:32,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5, entries=150, sequenceid=141, filesize=11.9 K 2024-11-22T23:16:32,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a67cd775c3b32775e1402006c75b11e in 1239ms, sequenceid=141, compaction requested=true 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:32,130 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:32,130 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:32,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:32,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:32,131 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:16:32,131 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:32,131 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:32,131 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:32,131 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/62263019653f4a01b0481cbdfbfb222a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.5 K 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:32,131 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:32,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,131 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:32,131 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:32,131 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 62263019653f4a01b0481cbdfbfb222a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317385659 2024-11-22T23:16:32,131 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/bc677982de8943be91a84e1e76d80063, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=37.8 K 2024-11-22T23:16:32,132 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ce3b400a90424fc0a93e20fd8dd22e07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732317388058 2024-11-22T23:16:32,132 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc677982de8943be91a84e1e76d80063, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317385659 2024-11-22T23:16:32,132 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46fdfa605d5542d3a5a3316b95633a63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732317388058 2024-11-22T23:16:32,132 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 13a8155aa8d94c48ad40eaea2c2361c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:32,132 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1a135fff19d44f3b723466f7a9a4fce, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:32,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/db547bb49288466996320161485109ef is 50, key is test_row_0/A:col10/1732317390930/Put/seqid=0 2024-11-22T23:16:32,137 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:32,137 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/db1cc92ccda543eb8f1eca8602d2b3cb is 50, key is test_row_0/A:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:32,139 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:32,139 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/368f5f6cf5a24c3bb41d390a6ad139c1 is 50, key is test_row_0/B:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:32,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742291_1467 (size=12151) 2024-11-22T23:16:32,142 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/db547bb49288466996320161485109ef 2024-11-22T23:16:32,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742292_1468 (size=12459) 2024-11-22T23:16:32,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742293_1469 (size=12459) 2024-11-22T23:16:32,151 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/db1cc92ccda543eb8f1eca8602d2b3cb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db1cc92ccda543eb8f1eca8602d2b3cb 2024-11-22T23:16:32,151 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/368f5f6cf5a24c3bb41d390a6ad139c1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/368f5f6cf5a24c3bb41d390a6ad139c1 2024-11-22T23:16:32,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e3bc3125974045ddb62a034d2a11fc9c is 50, key is test_row_0/B:col10/1732317390930/Put/seqid=0 2024-11-22T23:16:32,158 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into db1cc92ccda543eb8f1eca8602d2b3cb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:32,158 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into 368f5f6cf5a24c3bb41d390a6ad139c1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:32,158 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317392130; duration=0sec 2024-11-22T23:16:32,158 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317392130; duration=0sec 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:32,158 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:32,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742294_1470 (size=12151) 2024-11-22T23:16:32,160 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:32,160 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:32,160 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:32,161 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6f9283cd36874b7d8ea6d4aa06633fae, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.5 K 2024-11-22T23:16:32,161 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9283cd36874b7d8ea6d4aa06633fae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732317385659 2024-11-22T23:16:32,161 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2add79512a04024af90963470d199cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732317388058 2024-11-22T23:16:32,161 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3b8d956c75b4406af3bbb159e70e7f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:32,166 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:32,167 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/2796b1651e714dbcaf97429d52718475 is 50, key is test_row_0/C:col10/1732317390258/Put/seqid=0 2024-11-22T23:16:32,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742295_1471 (size=12459) 2024-11-22T23:16:32,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:32,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317452344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,356 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317452457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,559 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e3bc3125974045ddb62a034d2a11fc9c 2024-11-22T23:16:32,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/431f30537ec24a82a8f79fb80b9c327f is 50, key is test_row_0/C:col10/1732317390930/Put/seqid=0 2024-11-22T23:16:32,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742296_1472 (size=12151) 2024-11-22T23:16:32,572 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/431f30537ec24a82a8f79fb80b9c327f 2024-11-22T23:16:32,574 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/2796b1651e714dbcaf97429d52718475 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/2796b1651e714dbcaf97429d52718475 2024-11-22T23:16:32,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/db547bb49288466996320161485109ef as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef 2024-11-22T23:16:32,579 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T23:16:32,579 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into 2796b1651e714dbcaf97429d52718475(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e3bc3125974045ddb62a034d2a11fc9c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c 2024-11-22T23:16:32,579 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:32,579 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317392130; duration=0sec 2024-11-22T23:16:32,580 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:32,580 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:32,582 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T23:16:32,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/431f30537ec24a82a8f79fb80b9c327f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f 2024-11-22T23:16:32,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f, entries=150, sequenceid=160, filesize=11.9 K 2024-11-22T23:16:32,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8a67cd775c3b32775e1402006c75b11e in 455ms, sequenceid=160, compaction requested=false 2024-11-22T23:16:32,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:32,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:32,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-22T23:16:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-22T23:16:32,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-22T23:16:32,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-22T23:16:32,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.8310 sec 2024-11-22T23:16:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:32,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:32,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:32,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/d1f40490671e46ba971a2517fd0e1c51 is 50, key is test_row_0/A:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:32,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742297_1473 (size=14541) 2024-11-22T23:16:32,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317452782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317452897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-22T23:16:32,908 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:32,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-22T23:16:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:32,910 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:32,911 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:32,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:33,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/d1f40490671e46ba971a2517fd0e1c51 2024-11-22T23:16:33,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/55d95ffc70c44be686778190f1552b00 is 50, key is test_row_0/B:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:33,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742298_1474 (size=12151) 2024-11-22T23:16:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317453107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317453107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317453107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,113 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:33,113 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317453118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317453118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,124 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,125 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:33,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317453417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:33,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/55d95ffc70c44be686778190f1552b00 2024-11-22T23:16:33,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3923d1e6b2874f9cbbe563bca2805512 is 50, key is test_row_0/C:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:33,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742299_1475 (size=12151) 2024-11-22T23:16:33,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:33,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3923d1e6b2874f9cbbe563bca2805512 2024-11-22T23:16:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/d1f40490671e46ba971a2517fd0e1c51 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51 2024-11-22T23:16:33,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51, entries=200, sequenceid=177, filesize=14.2 K 2024-11-22T23:16:33,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/55d95ffc70c44be686778190f1552b00 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00 2024-11-22T23:16:33,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T23:16:33,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3923d1e6b2874f9cbbe563bca2805512 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512 2024-11-22T23:16:33,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512, entries=150, sequenceid=177, filesize=11.9 K 2024-11-22T23:16:33,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8a67cd775c3b32775e1402006c75b11e in 1242ms, sequenceid=177, compaction requested=true 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:33,924 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:33,924 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:33,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:33,924 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:33,924 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:33,925 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,925 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,925 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/368f5f6cf5a24c3bb41d390a6ad139c1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.9 K 2024-11-22T23:16:33,925 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db1cc92ccda543eb8f1eca8602d2b3cb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=38.2 K 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting db1cc92ccda543eb8f1eca8602d2b3cb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 368f5f6cf5a24c3bb41d390a6ad139c1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting db547bb49288466996320161485109ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732317390916 2024-11-22T23:16:33,925 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting e3bc3125974045ddb62a034d2a11fc9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732317390916 2024-11-22T23:16:33,926 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 55d95ffc70c44be686778190f1552b00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:33,926 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f40490671e46ba971a2517fd0e1c51, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:33,940 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:33,940 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:33,940 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/79b92fbf87344ec4acb4d3afbacc2a1d is 50, key is test_row_0/A:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:33,940 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/6489c713084445c885f5312f6825a4c5 is 50, key is test_row_0/B:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:33,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742300_1476 (size=12561) 2024-11-22T23:16:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742301_1477 (size=12561) 2024-11-22T23:16:33,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a is 50, key is test_row_0/A:col10/1732317393942/Put/seqid=0 2024-11-22T23:16:33,952 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/6489c713084445c885f5312f6825a4c5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/6489c713084445c885f5312f6825a4c5 2024-11-22T23:16:33,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742302_1478 (size=14541) 2024-11-22T23:16:33,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a 2024-11-22T23:16:33,958 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into 6489c713084445c885f5312f6825a4c5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:33,958 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:33,958 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317393924; duration=0sec 2024-11-22T23:16:33,958 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:33,958 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:33,959 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:33,961 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:33,961 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:33,961 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,961 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/2796b1651e714dbcaf97429d52718475, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=35.9 K 2024-11-22T23:16:33,961 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2796b1651e714dbcaf97429d52718475, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1732317390242 2024-11-22T23:16:33,961 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 431f30537ec24a82a8f79fb80b9c327f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732317390916 2024-11-22T23:16:33,962 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3923d1e6b2874f9cbbe563bca2805512, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:33,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/89025473095a448aa59e0323400284fc is 50, key is test_row_0/B:col10/1732317393942/Put/seqid=0 2024-11-22T23:16:33,970 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:33,970 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/64d350ea2bde4f04b1f2459637035684 is 50, key is test_row_0/C:col10/1732317392313/Put/seqid=0 2024-11-22T23:16:33,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742303_1479 (size=12151) 2024-11-22T23:16:33,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742304_1480 (size=12561) 2024-11-22T23:16:33,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:33,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:33,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317454010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:34,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:34,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317454128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:34,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317454338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:34,349 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/79b92fbf87344ec4acb4d3afbacc2a1d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/79b92fbf87344ec4acb4d3afbacc2a1d 2024-11-22T23:16:34,353 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into 79b92fbf87344ec4acb4d3afbacc2a1d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:34,353 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:34,353 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317393924; duration=0sec 2024-11-22T23:16:34,353 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:34,353 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:34,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/89025473095a448aa59e0323400284fc 2024-11-22T23:16:34,377 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/64d350ea2bde4f04b1f2459637035684 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/64d350ea2bde4f04b1f2459637035684 2024-11-22T23:16:34,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/41d00e0b723f421a82ea6923d2dd84fe is 50, key is test_row_0/C:col10/1732317393942/Put/seqid=0 2024-11-22T23:16:34,381 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into 64d350ea2bde4f04b1f2459637035684(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:34,381 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:34,381 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317393924; duration=0sec 2024-11-22T23:16:34,382 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:34,382 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:34,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742305_1481 (size=12151) 2024-11-22T23:16:34,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317454647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:34,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:34,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:34,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/41d00e0b723f421a82ea6923d2dd84fe 2024-11-22T23:16:34,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a 2024-11-22T23:16:34,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a, entries=200, sequenceid=196, filesize=14.2 K 2024-11-22T23:16:34,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/89025473095a448aa59e0323400284fc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc 2024-11-22T23:16:34,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T23:16:34,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/41d00e0b723f421a82ea6923d2dd84fe as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe 2024-11-22T23:16:34,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe, entries=150, sequenceid=196, filesize=11.9 K 2024-11-22T23:16:34,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 8a67cd775c3b32775e1402006c75b11e in 853ms, sequenceid=196, compaction requested=false 2024-11-22T23:16:34,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:34,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-22T23:16:34,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:34,894 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:34,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/666e7609f18543989dadd2e94eff594a is 50, key is test_row_0/A:col10/1732317393995/Put/seqid=0 2024-11-22T23:16:34,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742306_1482 (size=12151) 2024-11-22T23:16:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:35,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:35,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317455157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317455159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317455160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317455168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317455168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317455277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317455277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317455277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317455277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317455277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,300 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/666e7609f18543989dadd2e94eff594a 2024-11-22T23:16:35,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/82af1d29dd3c43f09762f0c42bbc346a is 50, key is test_row_0/B:col10/1732317393995/Put/seqid=0 2024-11-22T23:16:35,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742307_1483 (size=12151) 2024-11-22T23:16:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317455487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317455488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317455488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317455488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317455488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,709 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/82af1d29dd3c43f09762f0c42bbc346a 2024-11-22T23:16:35,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3468ec5f7c96433981d97ff307885337 is 50, key is test_row_0/C:col10/1732317393995/Put/seqid=0 2024-11-22T23:16:35,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742308_1484 (size=12151) 2024-11-22T23:16:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317455797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317455797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317455797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317455797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317455807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:35,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,121 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3468ec5f7c96433981d97ff307885337 2024-11-22T23:16:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/666e7609f18543989dadd2e94eff594a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a 2024-11-22T23:16:36,127 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T23:16:36,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/82af1d29dd3c43f09762f0c42bbc346a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a 2024-11-22T23:16:36,131 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T23:16:36,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/3468ec5f7c96433981d97ff307885337 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337 2024-11-22T23:16:36,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337, entries=150, sequenceid=212, filesize=11.9 K 2024-11-22T23:16:36,135 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8a67cd775c3b32775e1402006c75b11e in 1242ms, sequenceid=212, compaction requested=true 2024-11-22T23:16:36,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:36,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:36,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-22T23:16:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-22T23:16:36,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-22T23:16:36,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2260 sec 2024-11-22T23:16:36,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 3.2300 sec 2024-11-22T23:16:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:36,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:36,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:36,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/4a5f07fb181a4285b916c455342cf6d8 is 50, key is test_row_0/A:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:36,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742309_1485 (size=14541) 2024-11-22T23:16:36,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317456328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317456337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317456338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317456340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317456341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317456448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317456448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317456457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317456457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317456657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317456658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,661 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317456677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317456677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/4a5f07fb181a4285b916c455342cf6d8 2024-11-22T23:16:36,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/db7f7a4e611048439e99490910ffbb6c is 50, key is test_row_0/B:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:36,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742310_1486 (size=12151) 2024-11-22T23:16:36,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317456967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317456967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:36,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317456998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317456998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-22T23:16:37,068 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:37,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-22T23:16:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:37,070 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:37,070 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:37,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:37,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/db7f7a4e611048439e99490910ffbb6c 2024-11-22T23:16:37,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f9860c391671480f8726f687e7e3fa9a is 50, key is test_row_0/C:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:37,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742311_1487 (size=12151) 2024-11-22T23:16:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:37,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:16:37,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:37,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317457358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,367 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:16:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:37,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317457488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317457488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,495 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317457507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:37,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317457517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:37,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f9860c391671480f8726f687e7e3fa9a 2024-11-22T23:16:37,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/4a5f07fb181a4285b916c455342cf6d8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8 2024-11-22T23:16:37,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8, entries=200, sequenceid=232, filesize=14.2 K 2024-11-22T23:16:37,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/db7f7a4e611048439e99490910ffbb6c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c 2024-11-22T23:16:37,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T23:16:37,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f9860c391671480f8726f687e7e3fa9a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a 2024-11-22T23:16:37,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a, entries=150, sequenceid=232, filesize=11.9 K 2024-11-22T23:16:37,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8a67cd775c3b32775e1402006c75b11e in 1241ms, sequenceid=232, compaction requested=true 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:37,554 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:37,554 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:37,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:37,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,555 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,555 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/79b92fbf87344ec4acb4d3afbacc2a1d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=52.5 K 2024-11-22T23:16:37,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/6489c713084445c885f5312f6825a4c5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=47.9 K 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6489c713084445c885f5312f6825a4c5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b92fbf87344ec4acb4d3afbacc2a1d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 89025473095a448aa59e0323400284fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732317392726 2024-11-22T23:16:37,555 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e38bcdf1d9241e7a3fd869fcdc5bf3a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732317392726 2024-11-22T23:16:37,556 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 82af1d29dd3c43f09762f0c42bbc346a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732317393952 2024-11-22T23:16:37,556 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 666e7609f18543989dadd2e94eff594a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732317393952 2024-11-22T23:16:37,556 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting db7f7a4e611048439e99490910ffbb6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:37,556 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a5f07fb181a4285b916c455342cf6d8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:37,562 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:37,562 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:37,563 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/6bb312e176014eceaaf37f26c36de346 is 50, key is test_row_0/A:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:37,563 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/b51283ad052c44b2b053cb8bea800bac is 50, key is test_row_0/B:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:37,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742312_1488 (size=12697) 2024-11-22T23:16:37,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742313_1489 (size=12697) 2024-11-22T23:16:37,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,680 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:37,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/c5e82f167cff4bd6a48f579ab385e60b is 50, key is test_row_0/A:col10/1732317396339/Put/seqid=0 2024-11-22T23:16:37,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742314_1490 (size=12151) 2024-11-22T23:16:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:37,969 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/6bb312e176014eceaaf37f26c36de346 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/6bb312e176014eceaaf37f26c36de346 2024-11-22T23:16:37,975 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/b51283ad052c44b2b053cb8bea800bac as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/b51283ad052c44b2b053cb8bea800bac 2024-11-22T23:16:37,976 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into 6bb312e176014eceaaf37f26c36de346(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:37,976 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:37,976 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=12, startTime=1732317397554; duration=0sec 2024-11-22T23:16:37,976 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:37,976 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:37,976 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:16:37,978 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:16:37,978 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:37,978 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:37,978 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/64d350ea2bde4f04b1f2459637035684, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=47.9 K 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d350ea2bde4f04b1f2459637035684, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732317392313 2024-11-22T23:16:37,979 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into b51283ad052c44b2b053cb8bea800bac(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d00e0b723f421a82ea6923d2dd84fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732317392726 2024-11-22T23:16:37,979 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=12, startTime=1732317397554; duration=0sec 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:37,979 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3468ec5f7c96433981d97ff307885337, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732317393952 2024-11-22T23:16:37,980 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9860c391671480f8726f687e7e3fa9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:37,987 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#422 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:37,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/bf95be853092481da9daecadccb83a69 is 50, key is test_row_0/C:col10/1732317396312/Put/seqid=0 2024-11-22T23:16:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742315_1491 (size=12697) 2024-11-22T23:16:38,087 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/c5e82f167cff4bd6a48f579ab385e60b 2024-11-22T23:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/41150ec3ffbf44a29e0416422d005468 is 50, key is test_row_0/B:col10/1732317396339/Put/seqid=0 2024-11-22T23:16:38,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742316_1492 (size=12151) 2024-11-22T23:16:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:38,395 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/bf95be853092481da9daecadccb83a69 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/bf95be853092481da9daecadccb83a69 2024-11-22T23:16:38,398 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into bf95be853092481da9daecadccb83a69(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:38,398 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:38,398 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=12, startTime=1732317397554; duration=0sec 2024-11-22T23:16:38,398 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:38,398 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:38,499 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/41150ec3ffbf44a29e0416422d005468 2024-11-22T23:16:38,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/82150d582eb948fc9eecede741480d2d is 50, key is test_row_0/C:col10/1732317396339/Put/seqid=0 2024-11-22T23:16:38,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742317_1493 (size=12151) 2024-11-22T23:16:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:38,509 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/82150d582eb948fc9eecede741480d2d 2024-11-22T23:16:38,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/c5e82f167cff4bd6a48f579ab385e60b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b 2024-11-22T23:16:38,515 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b, entries=150, sequenceid=244, filesize=11.9 K 2024-11-22T23:16:38,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/41150ec3ffbf44a29e0416422d005468 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468 2024-11-22T23:16:38,519 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468, entries=150, sequenceid=244, filesize=11.9 K 2024-11-22T23:16:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/82150d582eb948fc9eecede741480d2d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d 2024-11-22T23:16:38,523 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d, entries=150, sequenceid=244, filesize=11.9 K 2024-11-22T23:16:38,524 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=46.96 KB/48090 for 8a67cd775c3b32775e1402006c75b11e in 844ms, sequenceid=244, compaction requested=false 2024-11-22T23:16:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:38,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-22T23:16:38,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-22T23:16:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:38,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:38,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:38,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4550 sec 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:38,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.4600 sec 2024-11-22T23:16:38,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/b05b98b4cf074df4b711e6ffa3164042 is 50, key is test_row_0/A:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:38,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742318_1494 (size=16981) 2024-11-22T23:16:38,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317458571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317398553, max timestamp=1732317398582Num uniques 859; 2024-11-22T23:16:38,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:38,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317398553, max timestamp=1732317398582Num uniques 859; 2024-11-22T23:16:38,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732317398553, max timestamp=1732317398582Num uniques 859; 2024-11-22T23:16:38,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-22T23:16:38,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317458627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317458637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317458638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317458697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317458747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317458747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317458757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317458917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/b05b98b4cf074df4b711e6ffa3164042 2024-11-22T23:16:38,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/725125674bba41ffafed382f0a058243 is 50, key is test_row_0/B:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:38,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742319_1495 (size=12201) 2024-11-22T23:16:38,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317458958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317458958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317458977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-22T23:16:39,217 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:39,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-22T23:16:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:39,219 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:39,220 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:39,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317459237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317459277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317459278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317459297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:39,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/725125674bba41ffafed382f0a058243 2024-11-22T23:16:39,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f00d963591264c1e815778f5e935b460 is 50, key is test_row_0/C:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:39,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742320_1496 (size=12201) 2024-11-22T23:16:39,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f00d963591264c1e815778f5e935b460 2024-11-22T23:16:39,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/b05b98b4cf074df4b711e6ffa3164042 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042 2024-11-22T23:16:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042, entries=250, sequenceid=258, filesize=16.6 K 2024-11-22T23:16:39,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/725125674bba41ffafed382f0a058243 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243 2024-11-22T23:16:39,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243, entries=150, sequenceid=258, filesize=11.9 K 2024-11-22T23:16:39,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/f00d963591264c1e815778f5e935b460 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460 2024-11-22T23:16:39,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460, entries=150, sequenceid=258, filesize=11.9 K 2024-11-22T23:16:39,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8a67cd775c3b32775e1402006c75b11e in 842ms, sequenceid=258, compaction requested=true 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:39,369 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:39,369 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:39,370 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:39,370 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:39,370 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/6bb312e176014eceaaf37f26c36de346, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=40.8 K 2024-11-22T23:16:39,370 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/b51283ad052c44b2b053cb8bea800bac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=36.2 K 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bb312e176014eceaaf37f26c36de346, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:39,370 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b51283ad052c44b2b053cb8bea800bac, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:39,371 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e82f167cff4bd6a48f579ab385e60b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732317396336 2024-11-22T23:16:39,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-22T23:16:39,372 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting b05b98b4cf074df4b711e6ffa3164042, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398508 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:39,372 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:39,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:39,373 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 41150ec3ffbf44a29e0416422d005468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732317396336 2024-11-22T23:16:39,373 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 725125674bba41ffafed382f0a058243, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398520 2024-11-22T23:16:39,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e1db2db862147c28708d982b97c7ff6 is 50, key is test_row_0/A:col10/1732317398620/Put/seqid=0 2024-11-22T23:16:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:39,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:39,388 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:39,389 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/84ddc0db4aff4c34887132805a891ceb is 50, key is test_row_0/A:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:39,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742321_1497 (size=12301) 2024-11-22T23:16:39,396 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#430 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:39,397 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/7e4c90644802482d942732fb24b6933d is 50, key is test_row_0/B:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:39,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742322_1498 (size=12849) 2024-11-22T23:16:39,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742323_1499 (size=12849) 2024-11-22T23:16:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317459455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317459577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317459758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,796 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e1db2db862147c28708d982b97c7ff6 2024-11-22T23:16:39,802 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/84ddc0db4aff4c34887132805a891ceb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/84ddc0db4aff4c34887132805a891ceb 2024-11-22T23:16:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/7ef21eaa5c0341d1a06a688b92ac7239 is 50, key is test_row_0/B:col10/1732317398620/Put/seqid=0 2024-11-22T23:16:39,808 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:16:39,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742324_1500 (size=12301) 2024-11-22T23:16:39,809 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into 84ddc0db4aff4c34887132805a891ceb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:39,809 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317399369; duration=0sec 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:39,809 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:39,809 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:39,810 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/bf95be853092481da9daecadccb83a69, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=36.2 K 2024-11-22T23:16:39,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317459797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317459797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,811 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf95be853092481da9daecadccb83a69, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732317395158 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317459797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,812 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82150d582eb948fc9eecede741480d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732317396336 2024-11-22T23:16:39,812 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00d963591264c1e815778f5e935b460, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398520 2024-11-22T23:16:39,812 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/7e4c90644802482d942732fb24b6933d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7e4c90644802482d942732fb24b6933d 2024-11-22T23:16:39,816 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into 7e4c90644802482d942732fb24b6933d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:39,816 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:39,816 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317399369; duration=0sec 2024-11-22T23:16:39,816 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:39,816 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:39,818 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:39,818 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/d159e4bb612f4c3baca3c6fba2ff2b3c is 50, key is test_row_0/C:col10/1732317398520/Put/seqid=0 2024-11-22T23:16:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742325_1501 (size=12849) 2024-11-22T23:16:39,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317459808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:39,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:40,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317460117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,126 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/7ef21eaa5c0341d1a06a688b92ac7239 2024-11-22T23:16:40,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/8c3f0e1ba9fa468194bc3eef7c71c697 is 50, key is test_row_0/C:col10/1732317398620/Put/seqid=0 2024-11-22T23:16:40,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742326_1502 (size=12301) 2024-11-22T23:16:40,225 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/d159e4bb612f4c3baca3c6fba2ff2b3c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/d159e4bb612f4c3baca3c6fba2ff2b3c 2024-11-22T23:16:40,228 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into d159e4bb612f4c3baca3c6fba2ff2b3c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:40,228 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:40,228 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317399369; duration=0sec 2024-11-22T23:16:40,229 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:40,229 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:40,618 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/8c3f0e1ba9fa468194bc3eef7c71c697 2024-11-22T23:16:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/3e1db2db862147c28708d982b97c7ff6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6 2024-11-22T23:16:40,624 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:16:40,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/7ef21eaa5c0341d1a06a688b92ac7239 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239 2024-11-22T23:16:40,627 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:16:40,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/8c3f0e1ba9fa468194bc3eef7c71c697 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697 2024-11-22T23:16:40,630 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697, entries=150, sequenceid=292, filesize=12.0 K 2024-11-22T23:16:40,631 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 8a67cd775c3b32775e1402006c75b11e in 1259ms, sequenceid=292, compaction requested=false 2024-11-22T23:16:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:40,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-22T23:16:40,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-22T23:16:40,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-22T23:16:40,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-22T23:16:40,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.4160 sec 2024-11-22T23:16:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:40,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:40,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:40,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/04d5fa3824e446a29cf33c6eb3268ab2 is 50, key is test_row_0/A:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:40,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742327_1503 (size=14741) 2024-11-22T23:16:40,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317460754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317460777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,783 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317460818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,820 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317460827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317460827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,832 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,832 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:40,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317460867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:40,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:41,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/04d5fa3824e446a29cf33c6eb3268ab2 2024-11-22T23:16:41,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/2c3b3b7ce29f4f65a71e081b0cb64d98 is 50, key is test_row_0/B:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:41,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742328_1504 (size=12301) 2024-11-22T23:16:41,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317461087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-22T23:16:41,368 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:41,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-22T23:16:41,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:41,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:41,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:41,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317461407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:41,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:41,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/2c3b3b7ce29f4f65a71e081b0cb64d98 2024-11-22T23:16:41,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/b0260e24dd3e4f4f9d43e833c78d57d3 is 50, key is test_row_0/C:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:41,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742329_1505 (size=12301) 2024-11-22T23:16:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:41,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:41,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:41,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:41,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:41,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:41,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/b0260e24dd3e4f4f9d43e833c78d57d3 2024-11-22T23:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/04d5fa3824e446a29cf33c6eb3268ab2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2 2024-11-22T23:16:41,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2, entries=200, sequenceid=307, filesize=14.4 K 2024-11-22T23:16:41,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/2c3b3b7ce29f4f65a71e081b0cb64d98 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98 2024-11-22T23:16:41,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98, entries=150, sequenceid=307, filesize=12.0 K 2024-11-22T23:16:41,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/b0260e24dd3e4f4f9d43e833c78d57d3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3 2024-11-22T23:16:41,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3, entries=150, sequenceid=307, filesize=12.0 K 2024-11-22T23:16:41,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8a67cd775c3b32775e1402006c75b11e in 1251ms, sequenceid=307, compaction requested=true 2024-11-22T23:16:41,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:41,894 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:41,894 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8a67cd775c3b32775e1402006c75b11e:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:41,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:41,895 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/A is initiating minor compaction (all files) 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/B is initiating minor compaction (all files) 2024-11-22T23:16:41,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/B in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,896 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/A in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,896 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/84ddc0db4aff4c34887132805a891ceb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=39.0 K 2024-11-22T23:16:41,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7e4c90644802482d942732fb24b6933d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=36.6 K 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4c90644802482d942732fb24b6933d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398520 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ddc0db4aff4c34887132805a891ceb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398520 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef21eaa5c0341d1a06a688b92ac7239, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732317398620 2024-11-22T23:16:41,896 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e1db2db862147c28708d982b97c7ff6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732317398620 2024-11-22T23:16:41,897 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04d5fa3824e446a29cf33c6eb3268ab2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317399427 2024-11-22T23:16:41,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3b3b7ce29f4f65a71e081b0cb64d98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317399427 2024-11-22T23:16:41,916 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:41,916 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/38c8d3d7e5654614b3c010495b2ce4f0 is 50, key is test_row_0/A:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:41,916 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:41,917 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/33974772374b4f9985b75e1b44def087 is 50, key is test_row_0/B:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742331_1507 (size=13051) 2024-11-22T23:16:41,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742330_1506 (size=13051) 2024-11-22T23:16:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:41,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-22T23:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:41,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:41,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/8903981575514cd0b5d415f8b7885a49 is 50, key is test_row_0/A:col10/1732317401933/Put/seqid=0 2024-11-22T23:16:41,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742332_1508 (size=14741) 2024-11-22T23:16:41,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:41,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:42,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317462003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317462117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,326 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/33974772374b4f9985b75e1b44def087 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/33974772374b4f9985b75e1b44def087 2024-11-22T23:16:42,328 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/38c8d3d7e5654614b3c010495b2ce4f0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/38c8d3d7e5654614b3c010495b2ce4f0 2024-11-22T23:16:42,332 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/B of 8a67cd775c3b32775e1402006c75b11e into 33974772374b4f9985b75e1b44def087(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:42,332 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:42,332 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/B, priority=13, startTime=1732317401894; duration=0sec 2024-11-22T23:16:42,332 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:42,332 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:B 2024-11-22T23:16:42,332 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:42,333 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/A of 8a67cd775c3b32775e1402006c75b11e into 38c8d3d7e5654614b3c010495b2ce4f0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:42,333 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/A, priority=13, startTime=1732317401894; duration=0sec 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37451 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 8a67cd775c3b32775e1402006c75b11e/C is initiating minor compaction (all files) 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:A 2024-11-22T23:16:42,333 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8a67cd775c3b32775e1402006c75b11e/C in TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,333 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/d159e4bb612f4c3baca3c6fba2ff2b3c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp, totalSize=36.6 K 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d159e4bb612f4c3baca3c6fba2ff2b3c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732317398520 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c3f0e1ba9fa468194bc3eef7c71c697, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732317398620 2024-11-22T23:16:42,333 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b0260e24dd3e4f4f9d43e833c78d57d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732317399427 2024-11-22T23:16:42,346 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8a67cd775c3b32775e1402006c75b11e#C#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:42,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317462338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,347 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/83a0d2b6f34d4af1bea4ff0a23095420 is 50, key is test_row_0/C:col10/1732317399453/Put/seqid=0 2024-11-22T23:16:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/8903981575514cd0b5d415f8b7885a49 2024-11-22T23:16:42,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742333_1509 (size=13051) 2024-11-22T23:16:42,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e84b8bc49f5b4dda969996257668f254 is 50, key is test_row_0/B:col10/1732317401933/Put/seqid=0 2024-11-22T23:16:42,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742334_1510 (size=12301) 2024-11-22T23:16:42,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:42,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317462657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,755 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/83a0d2b6f34d4af1bea4ff0a23095420 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/83a0d2b6f34d4af1bea4ff0a23095420 2024-11-22T23:16:42,759 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8a67cd775c3b32775e1402006c75b11e/C of 8a67cd775c3b32775e1402006c75b11e into 83a0d2b6f34d4af1bea4ff0a23095420(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:42,759 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:42,759 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., storeName=8a67cd775c3b32775e1402006c75b11e/C, priority=13, startTime=1732317401894; duration=0sec 2024-11-22T23:16:42,759 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:42,759 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8a67cd775c3b32775e1402006c75b11e:C 2024-11-22T23:16:42,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e84b8bc49f5b4dda969996257668f254 2024-11-22T23:16:42,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/6dba76e6249340a292d72d1411b254b0 is 50, key is test_row_0/C:col10/1732317401933/Put/seqid=0 2024-11-22T23:16:42,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742335_1511 (size=12301) 2024-11-22T23:16:42,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46698 deadline: 1732317462808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,815 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46694 deadline: 1732317462847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,855 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732317462858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46714 deadline: 1732317462858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:42,865 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,866 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 8a67cd775c3b32775e1402006c75b11e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:42,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:42,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:43,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:43,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:43,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:43,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/6dba76e6249340a292d72d1411b254b0 2024-11-22T23:16:43,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/8903981575514cd0b5d415f8b7885a49 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/8903981575514cd0b5d415f8b7885a49 2024-11-22T23:16:43,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/8903981575514cd0b5d415f8b7885a49, entries=200, sequenceid=328, filesize=14.4 K 2024-11-22T23:16:43,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:43,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46670 deadline: 1732317463177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 , the old value is region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:43,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., hostname=a3d87a7361f3,46715,1732317251834, seqNum=2 because the exception is null or not the one we care about 2024-11-22T23:16:43,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/e84b8bc49f5b4dda969996257668f254 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e84b8bc49f5b4dda969996257668f254 2024-11-22T23:16:43,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e84b8bc49f5b4dda969996257668f254, entries=150, sequenceid=328, filesize=12.0 K 2024-11-22T23:16:43,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/6dba76e6249340a292d72d1411b254b0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6dba76e6249340a292d72d1411b254b0 2024-11-22T23:16:43,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6dba76e6249340a292d72d1411b254b0, entries=150, sequenceid=328, filesize=12.0 K 2024-11-22T23:16:43,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8a67cd775c3b32775e1402006c75b11e in 1263ms, sequenceid=328, compaction requested=false 2024-11-22T23:16:43,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:43,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-22T23:16:43,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:43,201 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:43,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:43,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/cccf667114374cc5ba159bab3895ef27 is 50, key is test_row_0/A:col10/1732317401964/Put/seqid=0 2024-11-22T23:16:43,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742336_1512 (size=12301) 2024-11-22T23:16:43,302 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-22T23:16:43,302 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:43,302 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,302 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:43,304 INFO [Thread-1997 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1997. 2024-11-22T23:16:43,304 DEBUG [Thread-1997 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:43,304 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,304 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-22T23:16:43,304 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:43,304 DEBUG [Thread-1997 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,304 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:43,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:43,305 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,307 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-22T23:16:43,307 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:43,307 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,307 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:43,309 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-22T23:16:43,309 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:43,309 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,309 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:43,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:43,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:43,619 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/cccf667114374cc5ba159bab3895ef27 2024-11-22T23:16:43,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ee69d0908f234ff7bf2dee1d221f1781 is 50, key is test_row_0/B:col10/1732317401964/Put/seqid=0 2024-11-22T23:16:43,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742337_1513 (size=12301) 2024-11-22T23:16:44,037 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ee69d0908f234ff7bf2dee1d221f1781 2024-11-22T23:16:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c320b53c57334b4582848d813d6d66ca is 50, key is test_row_0/C:col10/1732317401964/Put/seqid=0 2024-11-22T23:16:44,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742338_1514 (size=12301) 2024-11-22T23:16:44,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:44,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. as already flushing 2024-11-22T23:16:44,189 INFO [Thread-1980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1980. 2024-11-22T23:16:44,189 DEBUG [Thread-1980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:44,189 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:44,189 DEBUG [Thread-1980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:44,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:44,447 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c320b53c57334b4582848d813d6d66ca 2024-11-22T23:16:44,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/cccf667114374cc5ba159bab3895ef27 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cccf667114374cc5ba159bab3895ef27 2024-11-22T23:16:44,455 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cccf667114374cc5ba159bab3895ef27, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:16:44,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/ee69d0908f234ff7bf2dee1d221f1781 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ee69d0908f234ff7bf2dee1d221f1781 2024-11-22T23:16:44,459 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ee69d0908f234ff7bf2dee1d221f1781, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:16:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/c320b53c57334b4582848d813d6d66ca as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c320b53c57334b4582848d813d6d66ca 2024-11-22T23:16:44,464 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c320b53c57334b4582848d813d6d66ca, entries=150, sequenceid=342, filesize=12.0 K 2024-11-22T23:16:44,465 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for 8a67cd775c3b32775e1402006c75b11e in 1263ms, sequenceid=342, compaction requested=true 2024-11-22T23:16:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 8a67cd775c3b32775e1402006c75b11e: 2024-11-22T23:16:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-22T23:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-22T23:16:44,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-22T23:16:44,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0960 sec 2024-11-22T23:16:44,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 3.1000 sec 2024-11-22T23:16:45,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-22T23:16:45,528 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:46,849 INFO [Thread-1984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1984. 2024-11-22T23:16:46,849 DEBUG [Thread-1984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:46,849 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,849 DEBUG [Thread-1984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,849 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:46,878 INFO [Thread-1978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1978. 2024-11-22T23:16:46,879 DEBUG [Thread-1978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:46,879 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,879 DEBUG [Thread-1978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:46,909 INFO [Thread-1986 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1986. 2024-11-22T23:16:46,909 INFO [Thread-1982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1982. 2024-11-22T23:16:46,909 DEBUG [Thread-1986 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:46,909 DEBUG [Thread-1982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:16:46,909 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,909 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,909 DEBUG [Thread-1982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,909 DEBUG [Thread-1986 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:46,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1218 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3648 rows 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1215 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3644 rows 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3670 rows 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3668 rows 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1216 2024-11-22T23:16:46,910 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3648 rows 2024-11-22T23:16:46,910 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:16:46,910 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:16:46,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:46,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:16:46,911 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:16:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:46,914 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317406913"}]},"ts":"1732317406913"} 2024-11-22T23:16:46,915 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:16:46,915 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:16:46,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:16:46,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, UNASSIGN}] 2024-11-22T23:16:46,917 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, UNASSIGN 2024-11-22T23:16:46,918 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8a67cd775c3b32775e1402006c75b11e, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:46,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, UNASSIGN because future has completed 2024-11-22T23:16:46,919 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:16:46,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:47,071 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 8a67cd775c3b32775e1402006c75b11e, disabling compactions & flushes 2024-11-22T23:16:47,072 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. after waiting 0 ms 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:47,072 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 8a67cd775c3b32775e1402006c75b11e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=A 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=B 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8a67cd775c3b32775e1402006c75b11e, store=C 2024-11-22T23:16:47,072 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:47,075 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/511703a2b9f4403ab40e66da009e6b5b is 50, key is test_row_0/A:col10/1732317406908/Put/seqid=0 2024-11-22T23:16:47,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742339_1515 (size=12301) 2024-11-22T23:16:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:47,479 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/511703a2b9f4403ab40e66da009e6b5b 2024-11-22T23:16:47,485 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/fc082947b970480db2afc35e27712acc is 50, key is test_row_0/B:col10/1732317406908/Put/seqid=0 2024-11-22T23:16:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742340_1516 (size=12301) 2024-11-22T23:16:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:47,888 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/fc082947b970480db2afc35e27712acc 2024-11-22T23:16:47,895 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/be991b04e64448419f9c56ffa49a6385 is 50, key is test_row_0/C:col10/1732317406908/Put/seqid=0 2024-11-22T23:16:47,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742341_1517 (size=12301) 2024-11-22T23:16:47,906 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/be991b04e64448419f9c56ffa49a6385 2024-11-22T23:16:47,910 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/A/511703a2b9f4403ab40e66da009e6b5b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/511703a2b9f4403ab40e66da009e6b5b 2024-11-22T23:16:47,914 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/511703a2b9f4403ab40e66da009e6b5b, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T23:16:47,915 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/B/fc082947b970480db2afc35e27712acc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/fc082947b970480db2afc35e27712acc 2024-11-22T23:16:47,919 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/fc082947b970480db2afc35e27712acc, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T23:16:47,920 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/.tmp/C/be991b04e64448419f9c56ffa49a6385 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/be991b04e64448419f9c56ffa49a6385 2024-11-22T23:16:47,924 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/be991b04e64448419f9c56ffa49a6385, entries=150, sequenceid=350, filesize=12.0 K 2024-11-22T23:16:47,925 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a67cd775c3b32775e1402006c75b11e in 852ms, sequenceid=350, compaction requested=true 2024-11-22T23:16:47,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cad8b537ac7b46b2b16a4523cbf9967a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/bc677982de8943be91a84e1e76d80063, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db1cc92ccda543eb8f1eca8602d2b3cb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/79b92fbf87344ec4acb4d3afbacc2a1d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/6bb312e176014eceaaf37f26c36de346, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/84ddc0db4aff4c34887132805a891ceb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2] to archive 2024-11-22T23:16:47,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:47,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/1b4ea58437164ab4923b9edc7a2eb4d6 2024-11-22T23:16:47,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc762ef4082a4a1eac3ab0244714754c 2024-11-22T23:16:47,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/dc4c57161ce64e0b86b8721a5e6093fc 2024-11-22T23:16:47,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cad8b537ac7b46b2b16a4523cbf9967a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cad8b537ac7b46b2b16a4523cbf9967a 2024-11-22T23:16:47,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/fc45aa5236814796b9d50ea1f08a4eb1 2024-11-22T23:16:47,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/bc677982de8943be91a84e1e76d80063 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/bc677982de8943be91a84e1e76d80063 2024-11-22T23:16:47,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e2937ebf14d49a4805e34d8d615af46 2024-11-22T23:16:47,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/46fdfa605d5542d3a5a3316b95633a63 2024-11-22T23:16:47,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/a1a135fff19d44f3b723466f7a9a4fce 2024-11-22T23:16:47,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db1cc92ccda543eb8f1eca8602d2b3cb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db1cc92ccda543eb8f1eca8602d2b3cb 2024-11-22T23:16:47,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/db547bb49288466996320161485109ef 2024-11-22T23:16:47,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/d1f40490671e46ba971a2517fd0e1c51 2024-11-22T23:16:47,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/79b92fbf87344ec4acb4d3afbacc2a1d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/79b92fbf87344ec4acb4d3afbacc2a1d 2024-11-22T23:16:47,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e38bcdf1d9241e7a3fd869fcdc5bf3a 2024-11-22T23:16:47,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/666e7609f18543989dadd2e94eff594a 2024-11-22T23:16:47,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/4a5f07fb181a4285b916c455342cf6d8 2024-11-22T23:16:47,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/6bb312e176014eceaaf37f26c36de346 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/6bb312e176014eceaaf37f26c36de346 2024-11-22T23:16:47,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/c5e82f167cff4bd6a48f579ab385e60b 2024-11-22T23:16:47,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/b05b98b4cf074df4b711e6ffa3164042 2024-11-22T23:16:47,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/84ddc0db4aff4c34887132805a891ceb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/84ddc0db4aff4c34887132805a891ceb 2024-11-22T23:16:47,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/3e1db2db862147c28708d982b97c7ff6 2024-11-22T23:16:47,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/04d5fa3824e446a29cf33c6eb3268ab2 2024-11-22T23:16:47,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f5a47f4793dc40d498c5defd62a63edc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/62263019653f4a01b0481cbdfbfb222a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/368f5f6cf5a24c3bb41d390a6ad139c1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/6489c713084445c885f5312f6825a4c5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/b51283ad052c44b2b053cb8bea800bac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7e4c90644802482d942732fb24b6933d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98] to archive 2024-11-22T23:16:47,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:47,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/223a986d2dec49edad0cb043d439be6c 2024-11-22T23:16:47,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f099c0470eca43409aede39cef7f0184 2024-11-22T23:16:47,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f5a47f4793dc40d498c5defd62a63edc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/f5a47f4793dc40d498c5defd62a63edc 2024-11-22T23:16:47,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/a91bdb0ec46e46f9b63af20f922b8fb5 2024-11-22T23:16:47,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bc7a4fe2d42449efbf8f95c953aa32ed 2024-11-22T23:16:47,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/62263019653f4a01b0481cbdfbfb222a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/62263019653f4a01b0481cbdfbfb222a 2024-11-22T23:16:47,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/bbd995632bc840ec87ae4bcd50d0fbcf 2024-11-22T23:16:47,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ce3b400a90424fc0a93e20fd8dd22e07 2024-11-22T23:16:47,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/368f5f6cf5a24c3bb41d390a6ad139c1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/368f5f6cf5a24c3bb41d390a6ad139c1 2024-11-22T23:16:47,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/13a8155aa8d94c48ad40eaea2c2361c0 2024-11-22T23:16:47,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e3bc3125974045ddb62a034d2a11fc9c 2024-11-22T23:16:47,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/6489c713084445c885f5312f6825a4c5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/6489c713084445c885f5312f6825a4c5 2024-11-22T23:16:47,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/55d95ffc70c44be686778190f1552b00 2024-11-22T23:16:47,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/89025473095a448aa59e0323400284fc 2024-11-22T23:16:47,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/82af1d29dd3c43f09762f0c42bbc346a 2024-11-22T23:16:47,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/b51283ad052c44b2b053cb8bea800bac to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/b51283ad052c44b2b053cb8bea800bac 2024-11-22T23:16:47,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/db7f7a4e611048439e99490910ffbb6c 2024-11-22T23:16:47,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/41150ec3ffbf44a29e0416422d005468 2024-11-22T23:16:47,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7e4c90644802482d942732fb24b6933d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7e4c90644802482d942732fb24b6933d 2024-11-22T23:16:47,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/725125674bba41ffafed382f0a058243 2024-11-22T23:16:47,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/7ef21eaa5c0341d1a06a688b92ac7239 2024-11-22T23:16:47,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/2c3b3b7ce29f4f65a71e081b0cb64d98 2024-11-22T23:16:47,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/9c812d21e68d4fb4a948dd5d3b41944e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6f9283cd36874b7d8ea6d4aa06633fae, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/2796b1651e714dbcaf97429d52718475, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/64d350ea2bde4f04b1f2459637035684, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/bf95be853092481da9daecadccb83a69, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/d159e4bb612f4c3baca3c6fba2ff2b3c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3] to archive 2024-11-22T23:16:47,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:16:47,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/73703a28962c402ea00d97ea6c672a04 2024-11-22T23:16:47,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/abd6aa07b4e24fd0b1c1a52740fa6f04 2024-11-22T23:16:47,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/9c812d21e68d4fb4a948dd5d3b41944e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/9c812d21e68d4fb4a948dd5d3b41944e 2024-11-22T23:16:47,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/821e8604798a49cb9047496d4d875974 2024-11-22T23:16:48,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/1a912c6378c04d899628c204d2f31ef8 2024-11-22T23:16:48,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6f9283cd36874b7d8ea6d4aa06633fae to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6f9283cd36874b7d8ea6d4aa06633fae 2024-11-22T23:16:48,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/fa36ed7c57db4c2593eef9bec96b4ae9 2024-11-22T23:16:48,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c2add79512a04024af90963470d199cb 2024-11-22T23:16:48,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/2796b1651e714dbcaf97429d52718475 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/2796b1651e714dbcaf97429d52718475 2024-11-22T23:16:48,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f3b8d956c75b4406af3bbb159e70e7f5 2024-11-22T23:16:48,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/431f30537ec24a82a8f79fb80b9c327f 2024-11-22T23:16:48,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/64d350ea2bde4f04b1f2459637035684 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/64d350ea2bde4f04b1f2459637035684 2024-11-22T23:16:48,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3923d1e6b2874f9cbbe563bca2805512 2024-11-22T23:16:48,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/41d00e0b723f421a82ea6923d2dd84fe 2024-11-22T23:16:48,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/3468ec5f7c96433981d97ff307885337 2024-11-22T23:16:48,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/bf95be853092481da9daecadccb83a69 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/bf95be853092481da9daecadccb83a69 2024-11-22T23:16:48,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f9860c391671480f8726f687e7e3fa9a 2024-11-22T23:16:48,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/82150d582eb948fc9eecede741480d2d 2024-11-22T23:16:48,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/d159e4bb612f4c3baca3c6fba2ff2b3c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/d159e4bb612f4c3baca3c6fba2ff2b3c 2024-11-22T23:16:48,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/f00d963591264c1e815778f5e935b460 2024-11-22T23:16:48,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/8c3f0e1ba9fa468194bc3eef7c71c697 2024-11-22T23:16:48,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/b0260e24dd3e4f4f9d43e833c78d57d3 2024-11-22T23:16:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:48,048 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=1 2024-11-22T23:16:48,048 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. 2024-11-22T23:16:48,048 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 8a67cd775c3b32775e1402006c75b11e: Waiting for close lock at 1732317407072Running coprocessor pre-close hooks at 1732317407072Disabling compacts and flushes for region at 1732317407072Disabling writes for close at 1732317407072Obtaining lock to block concurrent updates at 1732317407072Preparing flush snapshotting stores in 8a67cd775c3b32775e1402006c75b11e at 1732317407072Finished memstore snapshotting TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732317407072Flushing stores of TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e. at 1732317407073 (+1 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/A: creating writer at 1732317407073Flushing 8a67cd775c3b32775e1402006c75b11e/A: appending metadata at 1732317407075 (+2 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/A: closing flushed file at 1732317407075Flushing 8a67cd775c3b32775e1402006c75b11e/B: creating writer at 1732317407483 (+408 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/B: appending metadata at 1732317407485 (+2 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/B: closing flushed file at 1732317407485Flushing 8a67cd775c3b32775e1402006c75b11e/C: creating writer at 1732317407892 (+407 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/C: appending metadata at 1732317407895 (+3 ms)Flushing 8a67cd775c3b32775e1402006c75b11e/C: closing flushed file at 1732317407895Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74ee6079: reopening flushed file at 1732317407909 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6017c1a2: reopening flushed file at 1732317407914 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c13271: reopening flushed file at 1732317407919 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8a67cd775c3b32775e1402006c75b11e in 852ms, sequenceid=350, compaction requested=true at 1732317407925 (+6 ms)Writing region close event to WAL at 1732317408045 (+120 ms)Running coprocessor post-close hooks at 1732317408048 (+3 ms)Closed at 1732317408048 2024-11-22T23:16:48,050 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:48,050 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=8a67cd775c3b32775e1402006c75b11e, regionState=CLOSED 2024-11-22T23:16:48,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:48,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-22T23:16:48,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a67cd775c3b32775e1402006c75b11e, server=a3d87a7361f3,46715,1732317251834 in 1.1330 sec 2024-11-22T23:16:48,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-22T23:16:48,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8a67cd775c3b32775e1402006c75b11e, UNASSIGN in 1.1370 sec 2024-11-22T23:16:48,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-22T23:16:48,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1400 sec 2024-11-22T23:16:48,057 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317408057"}]},"ts":"1732317408057"} 2024-11-22T23:16:48,058 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:16:48,058 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:16:48,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-22T23:16:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-22T23:16:49,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:49,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:16:49,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,060 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T23:16:49,060 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,062 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:49,064 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/recovered.edits] 2024-11-22T23:16:49,066 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/38c8d3d7e5654614b3c010495b2ce4f0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/38c8d3d7e5654614b3c010495b2ce4f0 2024-11-22T23:16:49,067 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/511703a2b9f4403ab40e66da009e6b5b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/511703a2b9f4403ab40e66da009e6b5b 2024-11-22T23:16:49,068 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/8903981575514cd0b5d415f8b7885a49 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/8903981575514cd0b5d415f8b7885a49 2024-11-22T23:16:49,069 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cccf667114374cc5ba159bab3895ef27 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/A/cccf667114374cc5ba159bab3895ef27 2024-11-22T23:16:49,071 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/33974772374b4f9985b75e1b44def087 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/33974772374b4f9985b75e1b44def087 2024-11-22T23:16:49,071 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e84b8bc49f5b4dda969996257668f254 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/e84b8bc49f5b4dda969996257668f254 2024-11-22T23:16:49,072 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ee69d0908f234ff7bf2dee1d221f1781 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/ee69d0908f234ff7bf2dee1d221f1781 2024-11-22T23:16:49,073 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/fc082947b970480db2afc35e27712acc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/B/fc082947b970480db2afc35e27712acc 2024-11-22T23:16:49,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6dba76e6249340a292d72d1411b254b0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/6dba76e6249340a292d72d1411b254b0 2024-11-22T23:16:49,075 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/83a0d2b6f34d4af1bea4ff0a23095420 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/83a0d2b6f34d4af1bea4ff0a23095420 2024-11-22T23:16:49,076 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/be991b04e64448419f9c56ffa49a6385 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/be991b04e64448419f9c56ffa49a6385 2024-11-22T23:16:49,077 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c320b53c57334b4582848d813d6d66ca to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/C/c320b53c57334b4582848d813d6d66ca 2024-11-22T23:16:49,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/recovered.edits/353.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e/recovered.edits/353.seqid 2024-11-22T23:16:49,079 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/8a67cd775c3b32775e1402006c75b11e 2024-11-22T23:16:49,079 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:16:49,081 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,083 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:16:49,085 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:16:49,086 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,086 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:16:49,086 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317409086"}]},"ts":"9223372036854775807"} 2024-11-22T23:16:49,088 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:16:49,088 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8a67cd775c3b32775e1402006c75b11e, NAME => 'TestAcidGuarantees,,1732317381089.8a67cd775c3b32775e1402006c75b11e.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:16:49,088 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:16:49,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317409088"}]},"ts":"9223372036854775807"} 2024-11-22T23:16:49,090 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:16:49,091 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-22T23:16:49,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-22T23:16:49,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:16:49,168 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:49,176 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 251), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=384 (was 400), ProcessCount=11 (was 11), AvailableMemoryMB=1616 (was 654) - AvailableMemoryMB LEAK? - 2024-11-22T23:16:49,186 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=1615 2024-11-22T23:16:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:16:49,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:16:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:49,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:16:49,189 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:49,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-22T23:16:49,189 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:16:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:49,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742342_1518 (size=1061) 2024-11-22T23:16:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:49,596 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c 2024-11-22T23:16:49,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742343_1519 (size=53) 2024-11-22T23:16:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:50,000 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:16:50,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 924a94030c700005d123b5bb895e1d9d, disabling compactions & flushes 2024-11-22T23:16:50,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. after waiting 0 ms 2024-11-22T23:16:50,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 924a94030c700005d123b5bb895e1d9d: Waiting for close lock at 1732317410000Disabling compacts and flushes for region at 1732317410000Disabling writes for close at 1732317410001 (+1 ms)Writing region close event to WAL at 1732317410001Closed at 1732317410001 2024-11-22T23:16:50,002 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:16:50,002 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732317410002"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317410002"}]},"ts":"1732317410002"} 2024-11-22T23:16:50,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:16:50,004 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:16:50,004 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317410004"}]},"ts":"1732317410004"} 2024-11-22T23:16:50,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-22T23:16:50,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, ASSIGN}] 2024-11-22T23:16:50,007 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, ASSIGN 2024-11-22T23:16:50,007 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, ASSIGN; state=OFFLINE, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=false 2024-11-22T23:16:50,158 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:50,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, ASSIGN because future has completed 2024-11-22T23:16:50,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:50,314 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,314 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:16:50,314 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,314 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:16:50,315 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,315 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,316 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,317 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:50,317 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName A 2024-11-22T23:16:50,317 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:50,317 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:50,317 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,318 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:50,319 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName B 2024-11-22T23:16:50,319 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:50,319 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:50,319 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,320 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:50,320 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName C 2024-11-22T23:16:50,320 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:50,321 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:50,321 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,321 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,322 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,322 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,323 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,323 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,323 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:16:50,324 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,325 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:16:50,326 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 924a94030c700005d123b5bb895e1d9d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69933845, jitterRate=0.042095497250556946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:16:50,326 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:50,327 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 924a94030c700005d123b5bb895e1d9d: Running coprocessor pre-open hook at 1732317410315Writing region info on filesystem at 1732317410315Initializing all the Stores at 1732317410315Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317410315Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317410315Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317410315Cleaning up temporary data from old regions at 1732317410323 (+8 ms)Running coprocessor post-open hooks at 1732317410326 (+3 ms)Region opened successfully at 1732317410327 (+1 ms) 2024-11-22T23:16:50,328 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., pid=154, masterSystemTime=1732317410311 2024-11-22T23:16:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:50,329 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,329 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:50,329 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:50,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:50,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-22T23:16:50,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 in 171 msec 2024-11-22T23:16:50,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-22T23:16:50,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, ASSIGN in 326 msec 2024-11-22T23:16:50,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:16:50,334 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317410334"}]},"ts":"1732317410334"} 2024-11-22T23:16:50,335 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-22T23:16:50,336 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:16:50,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-22T23:16:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-22T23:16:51,339 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:51,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2cba0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:51,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:51,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:51,343 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a37b4ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:51,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:51,345 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:51,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f52b874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:51,348 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:51,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:51,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:16:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,34971,1732317250724 2024-11-22T23:16:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@440fa21e 2024-11-22T23:16:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:16:51,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:16:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-22T23:16:51,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:16:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-22T23:16:51,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742344_1520 (size=1097) 2024-11-22T23:16:51,765 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-22T23:16:51,765 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-22T23:16:51,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:16:51,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE}] 2024-11-22T23:16:51,772 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE 2024-11-22T23:16:51,773 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:51,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE because future has completed 2024-11-22T23:16:51,775 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:16:51,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:51,928 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:51,928 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:16:51,928 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 924a94030c700005d123b5bb895e1d9d, disabling compactions & flushes 2024-11-22T23:16:51,928 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:51,928 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:51,928 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. after waiting 0 ms 2024-11-22T23:16:51,929 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:51,933 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-22T23:16:51,934 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:51,934 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 924a94030c700005d123b5bb895e1d9d: Waiting for close lock at 1732317411928Running coprocessor pre-close hooks at 1732317411928Disabling compacts and flushes for region at 1732317411928Disabling writes for close at 1732317411928Writing region close event to WAL at 1732317411929 (+1 ms)Running coprocessor post-close hooks at 1732317411934 (+5 ms)Closed at 1732317411934 2024-11-22T23:16:51,934 WARN [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 924a94030c700005d123b5bb895e1d9d to self. 2024-11-22T23:16:51,936 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:51,937 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=CLOSED 2024-11-22T23:16:51,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:51,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-22T23:16:51,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 in 166 msec 2024-11-22T23:16:51,945 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE; state=CLOSED, location=a3d87a7361f3,46715,1732317251834; forceNewPlan=false, retain=true 2024-11-22T23:16:52,096 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=OPENING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE because future has completed 2024-11-22T23:16:52,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:16:52,258 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,258 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:16:52,258 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,258 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:16:52,259 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,259 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,261 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,262 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:52,263 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName A 2024-11-22T23:16:52,265 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:52,265 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:52,266 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,267 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:52,267 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName B 2024-11-22T23:16:52,267 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:52,268 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:52,268 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,268 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-22T23:16:52,269 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 924a94030c700005d123b5bb895e1d9d columnFamilyName C 2024-11-22T23:16:52,269 DEBUG [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:52,269 INFO [StoreOpener-924a94030c700005d123b5bb895e1d9d-1 {}] regionserver.HStore(327): Store=924a94030c700005d123b5bb895e1d9d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:16:52,270 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,270 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,271 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,272 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,273 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,273 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,273 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-22T23:16:52,274 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,275 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 924a94030c700005d123b5bb895e1d9d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70034008, jitterRate=0.04358804225921631}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-22T23:16:52,275 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,275 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 924a94030c700005d123b5bb895e1d9d: Running coprocessor pre-open hook at 1732317412259Writing region info on filesystem at 1732317412259Initializing all the Stores at 1732317412260 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317412260Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317412261 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317412261Cleaning up temporary data from old regions at 1732317412273 (+12 ms)Running coprocessor post-open hooks at 1732317412275 (+2 ms)Region opened successfully at 1732317412275 2024-11-22T23:16:52,276 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., pid=159, masterSystemTime=1732317412253 2024-11-22T23:16:52,277 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,277 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=OPEN, openSeqNum=5, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:16:52,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-22T23:16:52,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 in 180 msec 2024-11-22T23:16:52,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-22T23:16:52,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, REOPEN/MOVE in 510 msec 2024-11-22T23:16:52,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-22T23:16:52,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-11-22T23:16:52,285 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 930 msec 2024-11-22T23:16:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-22T23:16:52,286 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:52,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,287 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348a56d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9639cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,290 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651170ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,291 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3e3dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,292 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397c076f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,293 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174cba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a76487f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,296 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e9aead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,297 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4029628a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a0617c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,300 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e6dd80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72328188, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b331f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aef36f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c9d62b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,306 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc07449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,308 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573ac354, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7351e36d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,34971,-1 for getting cluster id 2024-11-22T23:16:52,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:16:52,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2f92b64c-3235-4726-bb52-42f565334fc6' 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2f92b64c-3235-4726-bb52-42f565334fc6" 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a37b97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,34971,-1] 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:16:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:16:52,312 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:16:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5435091a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:16:52,312 DEBUG [Thread-2332 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [Thread-2349 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [Thread-2347 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [Thread-2336 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [Thread-2340 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,313 DEBUG [Thread-2351 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,314 DEBUG [Thread-2343 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,314 DEBUG [Thread-2338 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,315 DEBUG [Thread-2334 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,315 DEBUG [Thread-2345 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:16:52,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:52,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,46715,1732317251834, seqNum=-1] 2024-11-22T23:16:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,318 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:16:52,319 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:52,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5] 2024-11-22T23:16:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:16:52,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:52,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317472341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317472342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317472343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317472344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317472344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122348ca7fa9a4d4cf5b2d50ae2f6461d72_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317412324/Put/seqid=0 2024-11-22T23:16:52,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742345_1521 (size=14594) 2024-11-22T23:16:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:52,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317472447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,450 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317472458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317472458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317472458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317472458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:52,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:52,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317472657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317472668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317472668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317472668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317472668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,762 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:52,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122348ca7fa9a4d4cf5b2d50ae2f6461d72_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122348ca7fa9a4d4cf5b2d50ae2f6461d72_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:52,765 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a37ba687ea1646c3b5594aee9a641ee5, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:52,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a37ba687ea1646c3b5594aee9a641ee5 is 175, key is test_row_0/A:col10/1732317412324/Put/seqid=0 2024-11-22T23:16:52,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742346_1522 (size=39549) 2024-11-22T23:16:52,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:52,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:52,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:52,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:52,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317472968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317472978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317472978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317472978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317472978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,174 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a37ba687ea1646c3b5594aee9a641ee5 2024-11-22T23:16:53,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d4241e668f054b80a2d6e1219f78b568 is 50, key is test_row_0/B:col10/1732317412324/Put/seqid=0 2024-11-22T23:16:53,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742347_1523 (size=12001) 2024-11-22T23:16:53,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:53,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:53,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317473478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:53,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:53,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:53,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:53,566 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:16:53,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d4241e668f054b80a2d6e1219f78b568 2024-11-22T23:16:53,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/8d3db44bb3d6469fbbb2626aaa3685b4 is 50, key is test_row_0/C:col10/1732317412324/Put/seqid=0 2024-11-22T23:16:53,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742348_1524 (size=12001) 2024-11-22T23:16:53,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/8d3db44bb3d6469fbbb2626aaa3685b4 2024-11-22T23:16:53,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a37ba687ea1646c3b5594aee9a641ee5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5 2024-11-22T23:16:53,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5, entries=200, sequenceid=17, filesize=38.6 K 2024-11-22T23:16:53,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d4241e668f054b80a2d6e1219f78b568 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568 2024-11-22T23:16:53,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568, entries=150, sequenceid=17, filesize=11.7 K 2024-11-22T23:16:53,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/8d3db44bb3d6469fbbb2626aaa3685b4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4 2024-11-22T23:16:53,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4, entries=150, sequenceid=17, filesize=11.7 K 2024-11-22T23:16:53,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 924a94030c700005d123b5bb895e1d9d in 1324ms, sequenceid=17, compaction requested=false 2024-11-22T23:16:53,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:53,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:53,697 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:53,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:53,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225befd91b95894750b65699d5cc279ff2_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317412342/Put/seqid=0 2024-11-22T23:16:53,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742349_1525 (size=12154) 2024-11-22T23:16:53,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:53,710 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225befd91b95894750b65699d5cc279ff2_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225befd91b95894750b65699d5cc279ff2_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/ea51667369674e85b798411b09fc24f8, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/ea51667369674e85b798411b09fc24f8 is 175, key is test_row_0/A:col10/1732317412342/Put/seqid=0 2024-11-22T23:16:53,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742350_1526 (size=30955) 2024-11-22T23:16:53,715 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=37, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/ea51667369674e85b798411b09fc24f8 2024-11-22T23:16:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1e3f8fe95ba4d09868183128b72ce54 is 50, key is test_row_0/B:col10/1732317412342/Put/seqid=0 2024-11-22T23:16:53,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742351_1527 (size=12001) 2024-11-22T23:16:53,741 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1e3f8fe95ba4d09868183128b72ce54 2024-11-22T23:16:53,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/19a1f1a7772847209d7b19ffeb029bcc is 50, key is test_row_0/C:col10/1732317412342/Put/seqid=0 2024-11-22T23:16:53,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742352_1528 (size=12001) 2024-11-22T23:16:54,151 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/19a1f1a7772847209d7b19ffeb029bcc 2024-11-22T23:16:54,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/ea51667369674e85b798411b09fc24f8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8 2024-11-22T23:16:54,157 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8, entries=150, sequenceid=37, filesize=30.2 K 2024-11-22T23:16:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1e3f8fe95ba4d09868183128b72ce54 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54 2024-11-22T23:16:54,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,160 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T23:16:54,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/19a1f1a7772847209d7b19ffeb029bcc as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc 2024-11-22T23:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,185 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc, entries=150, sequenceid=37, filesize=11.7 K 2024-11-22T23:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,186 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 924a94030c700005d123b5bb895e1d9d in 489ms, sequenceid=37, compaction requested=false 2024-11-22T23:16:54,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:54,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-22T23:16:54,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-22T23:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-22T23:16:54,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-11-22T23:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8740 sec 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-22T23:16:54,467 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-22T23:16:54,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,470 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:54,471 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:54,471 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:54,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:54,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e4a51b31f66e45fd88a7b40118344da1_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742353_1529 (size=19474) 2024-11-22T23:16:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,563 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:54,568 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e4a51b31f66e45fd88a7b40118344da1_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4a51b31f66e45fd88a7b40118344da1_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:54,570 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2012a0e9aa58400281b4589bafc3e6a0, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:54,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2012a0e9aa58400281b4589bafc3e6a0 is 175, key is test_row_0/A:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:54,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742354_1530 (size=56729) 2024-11-22T23:16:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317414565, max timestamp=1732317414575Num uniques 859; 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317414565, max timestamp=1732317414575Num uniques 859; 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317414565, max timestamp=1732317414575Num uniques 859; 2024-11-22T23:16:54,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:54,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317474594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317474595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317474595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317474596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317474597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:54,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:54,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317474708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317474708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317474708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317474708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317474708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317474917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317474917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317474917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317474917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317474917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:54,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:54,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:54,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:54,976 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=47, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2012a0e9aa58400281b4589bafc3e6a0 2024-11-22T23:16:54,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/85019661c49d40e1abc83f61f0255589 is 50, key is test_row_0/B:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742355_1531 (size=9657) 2024-11-22T23:16:54,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/85019661c49d40e1abc83f61f0255589 2024-11-22T23:16:54,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/a6a1b58d2ad14514991b11e483daa47a is 50, key is test_row_0/C:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:55,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742356_1532 (size=9657) 2024-11-22T23:16:55,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317475227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317475228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317475228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317475228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317475228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:55,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/a6a1b58d2ad14514991b11e483daa47a 2024-11-22T23:16:55,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2012a0e9aa58400281b4589bafc3e6a0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0 2024-11-22T23:16:55,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0, entries=300, sequenceid=47, filesize=55.4 K 2024-11-22T23:16:55,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/85019661c49d40e1abc83f61f0255589 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589 2024-11-22T23:16:55,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589, entries=100, sequenceid=47, filesize=9.4 K 2024-11-22T23:16:55,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/a6a1b58d2ad14514991b11e483daa47a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a 2024-11-22T23:16:55,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a, entries=100, sequenceid=47, filesize=9.4 K 2024-11-22T23:16:55,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 924a94030c700005d123b5bb895e1d9d in 890ms, sequenceid=47, compaction requested=true 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:55,424 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:55,424 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:55,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:55,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:55,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:16:55,425 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,425 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=124.3 K 2024-11-22T23:16:55,425 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:55,425 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,425 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0] 2024-11-22T23:16:55,425 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:16:55,425 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,426 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=32.9 K 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting a37ba687ea1646c3b5594aee9a641ee5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732317412322 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d4241e668f054b80a2d6e1219f78b568, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732317412324 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea51667369674e85b798411b09fc24f8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732317412337 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e3f8fe95ba4d09868183128b72ce54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732317412337 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2012a0e9aa58400281b4589bafc3e6a0, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317414520 2024-11-22T23:16:55,426 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 85019661c49d40e1abc83f61f0255589, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317414531 2024-11-22T23:16:55,431 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:55,432 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:55,433 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d9eabc642e4a4d51b5e1c6221893d753 is 50, key is test_row_0/B:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:55,435 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411222c127bcf3541404789505715ca7fdeac_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:55,437 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411222c127bcf3541404789505715ca7fdeac_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:55,437 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222c127bcf3541404789505715ca7fdeac_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:55,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742358_1534 (size=4469) 2024-11-22T23:16:55,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742357_1533 (size=12104) 2024-11-22T23:16:55,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-22T23:16:55,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,544 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:55,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112251330bac21914dd8b22b6049bba120fd_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317414594/Put/seqid=0 2024-11-22T23:16:55,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742359_1535 (size=12154) 2024-11-22T23:16:55,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:55,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317475774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317475774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317475775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317475779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317475780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,849 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#459 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:55,849 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/56fba3496ac44c9f88470bdf35ea07f1 is 175, key is test_row_0/A:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:55,860 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d9eabc642e4a4d51b5e1c6221893d753 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d9eabc642e4a4d51b5e1c6221893d753 2024-11-22T23:16:55,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742360_1536 (size=31165) 2024-11-22T23:16:55,866 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into d9eabc642e4a4d51b5e1c6221893d753(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:55,866 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:55,866 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=13, startTime=1732317415424; duration=0sec 2024-11-22T23:16:55,867 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:55,867 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:16:55,867 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:55,868 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:55,868 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:16:55,868 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:55,868 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=32.9 K 2024-11-22T23:16:55,869 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d3db44bb3d6469fbbb2626aaa3685b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732317412324 2024-11-22T23:16:55,870 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 19a1f1a7772847209d7b19ffeb029bcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732317412337 2024-11-22T23:16:55,870 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/56fba3496ac44c9f88470bdf35ea07f1 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1 2024-11-22T23:16:55,871 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a6a1b58d2ad14514991b11e483daa47a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317414531 2024-11-22T23:16:55,875 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 56fba3496ac44c9f88470bdf35ea07f1(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:55,875 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:55,875 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=13, startTime=1732317415424; duration=0sec 2024-11-22T23:16:55,876 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:55,876 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:16:55,881 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:55,881 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b345b7d73c14cfe9c36fe6af0ee3ef0 is 50, key is test_row_0/C:col10/1732317414533/Put/seqid=0 2024-11-22T23:16:55,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317475888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317475888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317475888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317475888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742361_1537 (size=12104) 2024-11-22T23:16:55,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317475897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:55,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:55,958 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112251330bac21914dd8b22b6049bba120fd_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112251330bac21914dd8b22b6049bba120fd_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:55,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/767b929463e84143b5f5fd151258eabe, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:55,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/767b929463e84143b5f5fd151258eabe is 175, key is test_row_0/A:col10/1732317414594/Put/seqid=0 2024-11-22T23:16:55,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742362_1538 (size=30955) 2024-11-22T23:16:56,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317476098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317476098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317476098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317476098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317476107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,108 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,303 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b345b7d73c14cfe9c36fe6af0ee3ef0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b345b7d73c14cfe9c36fe6af0ee3ef0 2024-11-22T23:16:56,306 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into 3b345b7d73c14cfe9c36fe6af0ee3ef0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:16:56,306 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:56,306 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=13, startTime=1732317415424; duration=0sec 2024-11-22T23:16:56,307 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:56,307 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:16:56,364 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/767b929463e84143b5f5fd151258eabe 2024-11-22T23:16:56,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/7433e489876e4831a1aece7b9e151c62 is 50, key is test_row_0/B:col10/1732317414594/Put/seqid=0 2024-11-22T23:16:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742363_1539 (size=12001) 2024-11-22T23:16:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317476408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317476408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317476408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317476408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317476417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:56,786 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/7433e489876e4831a1aece7b9e151c62 2024-11-22T23:16:56,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/4f2e5795e0ae43b9bbee1081abfb8eac is 50, key is test_row_0/C:col10/1732317414594/Put/seqid=0 2024-11-22T23:16:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742364_1540 (size=12001) 2024-11-22T23:16:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317476917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317476917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317476917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317476917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:56,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:56,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317476938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:57,195 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/4f2e5795e0ae43b9bbee1081abfb8eac 2024-11-22T23:16:57,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/767b929463e84143b5f5fd151258eabe as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe 2024-11-22T23:16:57,202 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe, entries=150, sequenceid=83, filesize=30.2 K 2024-11-22T23:16:57,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/7433e489876e4831a1aece7b9e151c62 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62 2024-11-22T23:16:57,205 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T23:16:57,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/4f2e5795e0ae43b9bbee1081abfb8eac as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac 2024-11-22T23:16:57,208 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac, entries=150, sequenceid=83, filesize=11.7 K 2024-11-22T23:16:57,209 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 924a94030c700005d123b5bb895e1d9d in 1665ms, sequenceid=83, compaction requested=false 2024-11-22T23:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:57,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-22T23:16:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-22T23:16:57,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-22T23:16:57,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7390 sec 2024-11-22T23:16:57,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7440 sec 2024-11-22T23:16:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:57,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:57,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:16:57,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122cb95cd380a784ee4815fdeb98facfa86_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317415741/Put/seqid=0 2024-11-22T23:16:57,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742365_1541 (size=12154) 2024-11-22T23:16:57,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317477988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:57,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317477988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,994 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317477989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317417942, max timestamp=1732317417993Num uniques 859; 2024-11-22T23:16:57,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:57,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317417942, max timestamp=1732317417993Num uniques 859; 2024-11-22T23:16:57,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:57,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317417942, max timestamp=1732317417993Num uniques 859; 2024-11-22T23:16:57,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:16:58,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317478051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317478056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317478097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317478097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317478098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317478157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317478167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317478307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317478308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317478308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,339 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:16:58,342 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122cb95cd380a784ee4815fdeb98facfa86_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122cb95cd380a784ee4815fdeb98facfa86_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:16:58,342 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4c8db159a29b4f5da7686a7a734afaf3, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:58,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4c8db159a29b4f5da7686a7a734afaf3 is 175, key is test_row_0/A:col10/1732317415741/Put/seqid=0 2024-11-22T23:16:58,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742366_1542 (size=30955) 2024-11-22T23:16:58,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317478367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317478378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317478617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317478617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317478617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-22T23:16:58,648 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:16:58,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:16:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-22T23:16:58,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:16:58,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:16:58,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:16:58,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:16:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317478677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:58,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317478687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:58,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:58,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4c8db159a29b4f5da7686a7a734afaf3 2024-11-22T23:16:58,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d6d9d152878c46f799bd5c5215928c78 is 50, key is test_row_0/B:col10/1732317415741/Put/seqid=0 2024-11-22T23:16:58,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742367_1543 (size=12001) 2024-11-22T23:16:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:16:58,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:58,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:58,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:16:59,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317479128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:59,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317479128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:59,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317479128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:59,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d6d9d152878c46f799bd5c5215928c78 2024-11-22T23:16:59,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ef01c683540431088b8c54558ea2adf is 50, key is test_row_0/C:col10/1732317415741/Put/seqid=0 2024-11-22T23:16:59,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742368_1544 (size=12001) 2024-11-22T23:16:59,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:59,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317479187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,193 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:59,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:16:59,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317479198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:16:59,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:16:59,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:16:59,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:16:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ef01c683540431088b8c54558ea2adf 2024-11-22T23:16:59,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:16:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:16:59,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4c8db159a29b4f5da7686a7a734afaf3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3 2024-11-22T23:16:59,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3, entries=150, sequenceid=97, filesize=30.2 K 2024-11-22T23:16:59,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/d6d9d152878c46f799bd5c5215928c78 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78 2024-11-22T23:16:59,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78, entries=150, sequenceid=97, filesize=11.7 K 2024-11-22T23:16:59,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ef01c683540431088b8c54558ea2adf as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf 2024-11-22T23:16:59,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf, entries=150, sequenceid=97, filesize=11.7 K 2024-11-22T23:16:59,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 924a94030c700005d123b5bb895e1d9d in 1649ms, sequenceid=97, compaction requested=true 2024-11-22T23:16:59,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:16:59,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:16:59,578 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:59,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:59,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:16:59,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:16:59,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:16:59,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93075 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:16:59,579 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,579 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,579 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=90.9 K 2024-11-22T23:16:59,579 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d9eabc642e4a4d51b5e1c6221893d753, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.3 K 2024-11-22T23:16:59,579 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,579 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3] 2024-11-22T23:16:59,580 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56fba3496ac44c9f88470bdf35ea07f1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317412337 2024-11-22T23:16:59,580 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d9eabc642e4a4d51b5e1c6221893d753, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317412337 2024-11-22T23:16:59,580 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 767b929463e84143b5f5fd151258eabe, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732317414593 2024-11-22T23:16:59,580 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7433e489876e4831a1aece7b9e151c62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732317414593 2024-11-22T23:16:59,581 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8db159a29b4f5da7686a7a734afaf3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:16:59,581 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting d6d9d152878c46f799bd5c5215928c78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:16:59,585 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:59,587 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:16:59,587 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/148f8172f32145a5838775d09eb31520 is 50, key is test_row_0/B:col10/1732317415741/Put/seqid=0 2024-11-22T23:16:59,590 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122daf018d0068d4dfe962e658704b19ed2_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:59,592 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122daf018d0068d4dfe962e658704b19ed2_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:59,593 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122daf018d0068d4dfe962e658704b19ed2_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:16:59,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742369_1545 (size=12207) 2024-11-22T23:16:59,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742370_1546 (size=4469) 2024-11-22T23:16:59,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:16:59,719 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:16:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:16:59,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221f000caa17eb40b2990240e4fe77030b_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317418054/Put/seqid=0 2024-11-22T23:16:59,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742371_1547 (size=12304) 2024-11-22T23:16:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:17:00,003 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/148f8172f32145a5838775d09eb31520 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/148f8172f32145a5838775d09eb31520 2024-11-22T23:17:00,006 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:00,006 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/7c499021a8e6432191140741e98d981f is 175, key is test_row_0/A:col10/1732317415741/Put/seqid=0 2024-11-22T23:17:00,007 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into 148f8172f32145a5838775d09eb31520(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:00,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:00,007 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=13, startTime=1732317419578; duration=0sec 2024-11-22T23:17:00,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:00,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:17:00,007 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:00,008 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:00,008 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:17:00,009 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:00,009 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b345b7d73c14cfe9c36fe6af0ee3ef0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.3 K 2024-11-22T23:17:00,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b345b7d73c14cfe9c36fe6af0ee3ef0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732317412337 2024-11-22T23:17:00,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f2e5795e0ae43b9bbee1081abfb8eac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732317414593 2024-11-22T23:17:00,009 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef01c683540431088b8c54558ea2adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:17:00,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742372_1548 (size=31161) 2024-11-22T23:17:00,018 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/7c499021a8e6432191140741e98d981f as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f 2024-11-22T23:17:00,023 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 7c499021a8e6432191140741e98d981f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:00,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:00,023 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=13, startTime=1732317419578; duration=0sec 2024-11-22T23:17:00,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:00,023 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:17:00,031 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:00,031 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0465c7998ffe4007a57b65a85621cca0 is 50, key is test_row_0/C:col10/1732317415741/Put/seqid=0 2024-11-22T23:17:00,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742373_1549 (size=12207) 2024-11-22T23:17:00,071 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0465c7998ffe4007a57b65a85621cca0 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0465c7998ffe4007a57b65a85621cca0 2024-11-22T23:17:00,076 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into 0465c7998ffe4007a57b65a85621cca0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:00,076 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:00,076 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=13, startTime=1732317419578; duration=0sec 2024-11-22T23:17:00,076 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:00,076 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:17:00,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:00,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:00,146 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221f000caa17eb40b2990240e4fe77030b_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221f000caa17eb40b2990240e4fe77030b_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/3635c8d3111a4a4790250f3207c50cb6, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/3635c8d3111a4a4790250f3207c50cb6 is 175, key is test_row_0/A:col10/1732317418054/Put/seqid=0 2024-11-22T23:17:00,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742374_1550 (size=31105) 2024-11-22T23:17:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317480155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317480157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317480159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317480197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,200 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317480207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,211 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317480267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317480478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317480478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317480478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,557 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/3635c8d3111a4a4790250f3207c50cb6 2024-11-22T23:17:00,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/06cade093d404e17892aa21b8a8bf25b is 50, key is test_row_0/B:col10/1732317418054/Put/seqid=0 2024-11-22T23:17:00,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742375_1551 (size=12151) 2024-11-22T23:17:00,566 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/06cade093d404e17892aa21b8a8bf25b 2024-11-22T23:17:00,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b000df59351405f957ff10f1b9e376d is 50, key is test_row_0/C:col10/1732317418054/Put/seqid=0 2024-11-22T23:17:00,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742376_1552 (size=12151) 2024-11-22T23:17:00,581 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b000df59351405f957ff10f1b9e376d 2024-11-22T23:17:00,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/3635c8d3111a4a4790250f3207c50cb6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6 2024-11-22T23:17:00,586 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6, entries=150, sequenceid=132, filesize=30.4 K 2024-11-22T23:17:00,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/06cade093d404e17892aa21b8a8bf25b as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b 2024-11-22T23:17:00,589 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b, entries=150, sequenceid=132, filesize=11.9 K 2024-11-22T23:17:00,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b000df59351405f957ff10f1b9e376d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d 2024-11-22T23:17:00,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d, entries=150, sequenceid=132, filesize=11.9 K 2024-11-22T23:17:00,593 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 924a94030c700005d123b5bb895e1d9d in 874ms, sequenceid=132, compaction requested=false 2024-11-22T23:17:00,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:00,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-22T23:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-22T23:17:00,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-22T23:17:00,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9450 sec 2024-11-22T23:17:00,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.9480 sec 2024-11-22T23:17:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:00,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-22T23:17:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:00,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:00,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225cac39838c54436489d77c6e0b706976_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:00,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742377_1553 (size=14794) 2024-11-22T23:17:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-22T23:17:00,798 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:00,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-22T23:17:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:00,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:17:00,800 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:17:00,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:17:00,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317480812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317480814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317480816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:00,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317480927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317480928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:00,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317480928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:00,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:00,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317481137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317481137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317481137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:01,201 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225cac39838c54436489d77c6e0b706976_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225cac39838c54436489d77c6e0b706976_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:01,206 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/f91806e8f6214ebabbc34b30ae372a72, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:01,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/f91806e8f6214ebabbc34b30ae372a72 is 175, key is test_row_0/A:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:01,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742378_1554 (size=39749) 2024-11-22T23:17:01,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317481447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317481448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317481448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/f91806e8f6214ebabbc34b30ae372a72 2024-11-22T23:17:01,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/bbe1f78f2e404790ba37c987b0767ba4 is 50, key is test_row_0/B:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:01,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742379_1555 (size=12151) 2024-11-22T23:17:01,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:01,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317481962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317481962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317481962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:01,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:02,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/bbe1f78f2e404790ba37c987b0767ba4 2024-11-22T23:17:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/52109c4339c94bec9a1a312ca39f38e2 is 50, key is test_row_0/C:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:02,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742380_1556 (size=12151) 2024-11-22T23:17:02,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:02,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317482217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:02,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317482217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:02,220 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:02,221 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_2' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:02,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:02,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/52109c4339c94bec9a1a312ca39f38e2 2024-11-22T23:17:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/f91806e8f6214ebabbc34b30ae372a72 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72 2024-11-22T23:17:02,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72, entries=200, sequenceid=147, filesize=38.8 K 2024-11-22T23:17:02,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/bbe1f78f2e404790ba37c987b0767ba4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4 2024-11-22T23:17:02,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4, entries=150, sequenceid=147, filesize=11.9 K 2024-11-22T23:17:02,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/52109c4339c94bec9a1a312ca39f38e2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2 2024-11-22T23:17:02,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2, entries=150, sequenceid=147, filesize=11.9 K 2024-11-22T23:17:02,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 924a94030c700005d123b5bb895e1d9d in 1672ms, sequenceid=147, compaction requested=true 2024-11-22T23:17:02,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:02,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:17:02,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:17:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:17:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:17:02,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-22T23:17:02,461 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:02,461 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:02,462 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:02,462 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:02,462 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:17:02,462 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:17:02,462 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,462 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,462 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=99.6 K 2024-11-22T23:17:02,462 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0465c7998ffe4007a57b65a85621cca0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.7 K 2024-11-22T23:17:02,462 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,462 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72] 2024-11-22T23:17:02,463 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c499021a8e6432191140741e98d981f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:17:02,463 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0465c7998ffe4007a57b65a85621cca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:17:02,463 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3635c8d3111a4a4790250f3207c50cb6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732317418048 2024-11-22T23:17:02,463 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b000df59351405f957ff10f1b9e376d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732317418048 2024-11-22T23:17:02,464 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91806e8f6214ebabbc34b30ae372a72, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:02,464 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 52109c4339c94bec9a1a312ca39f38e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:02,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:02,485 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:02,494 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:02,494 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/d747995948f74275ab4fa608390268a6 is 50, key is test_row_0/C:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:02,513 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411221d4f842e454a4db68ce11c4b99161445_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:02,515 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411221d4f842e454a4db68ce11c4b99161445_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:02,515 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411221d4f842e454a4db68ce11c4b99161445_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112264a1a9fa5601437a896c50dd31447957_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317420815/Put/seqid=0 2024-11-22T23:17:02,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742381_1557 (size=12459) 2024-11-22T23:17:02,542 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/d747995948f74275ab4fa608390268a6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/d747995948f74275ab4fa608390268a6 2024-11-22T23:17:02,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742382_1558 (size=4469) 2024-11-22T23:17:02,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742383_1559 (size=12304) 2024-11-22T23:17:02,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:02,552 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into d747995948f74275ab4fa608390268a6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:02,552 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:02,552 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=13, startTime=1732317422461; duration=0sec 2024-11-22T23:17:02,552 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:02,552 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:17:02,552 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:02,554 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:02,554 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:17:02,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:02,555 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/148f8172f32145a5838775d09eb31520, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.7 K 2024-11-22T23:17:02,555 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112264a1a9fa5601437a896c50dd31447957_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264a1a9fa5601437a896c50dd31447957_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:02,556 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 148f8172f32145a5838775d09eb31520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732317415741 2024-11-22T23:17:02,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a9c09dee496b456ca04081960a8712d9, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:02,556 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#476 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:02,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a9c09dee496b456ca04081960a8712d9 is 175, key is test_row_0/A:col10/1732317420815/Put/seqid=0 2024-11-22T23:17:02,557 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/25853199ebbc405eb5ff99b11ca2294e is 175, key is test_row_0/A:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:02,557 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 06cade093d404e17892aa21b8a8bf25b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732317418048 2024-11-22T23:17:02,558 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe1f78f2e404790ba37c987b0767ba4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:02,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742384_1560 (size=31105) 2024-11-22T23:17:02,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742385_1561 (size=31413) 2024-11-22T23:17:02,578 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a9c09dee496b456ca04081960a8712d9 2024-11-22T23:17:02,586 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:02,586 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/071008cb46b443f1b6a3e9718b116cdb is 50, key is test_row_0/B:col10/1732317420787/Put/seqid=0 2024-11-22T23:17:02,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/cbb5aecdca104d48aa0b427c293c058a is 50, key is test_row_0/B:col10/1732317420815/Put/seqid=0 2024-11-22T23:17:02,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742386_1562 (size=12459) 2024-11-22T23:17:02,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742387_1563 (size=12151) 2024-11-22T23:17:02,595 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/071008cb46b443f1b6a3e9718b116cdb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/071008cb46b443f1b6a3e9718b116cdb 2024-11-22T23:17:02,599 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into 071008cb46b443f1b6a3e9718b116cdb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:02,599 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:02,599 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=13, startTime=1732317422460; duration=0sec 2024-11-22T23:17:02,599 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:02,599 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:17:02,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:02,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:02,984 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/25853199ebbc405eb5ff99b11ca2294e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e 2024-11-22T23:17:02,992 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 25853199ebbc405eb5ff99b11ca2294e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:02,992 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:02,992 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=13, startTime=1732317422460; duration=0sec 2024-11-22T23:17:02,993 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:02,993 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:17:02,993 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/cbb5aecdca104d48aa0b427c293c058a 2024-11-22T23:17:02,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317482990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:02,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317482993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,995 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:02,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317482995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:02,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/87c98133cad645109d040c38ff436632 is 50, key is test_row_0/C:col10/1732317420815/Put/seqid=0 2024-11-22T23:17:03,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742388_1564 (size=12151) 2024-11-22T23:17:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317483097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317483097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317483107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317483307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317483307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317483317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,423 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/87c98133cad645109d040c38ff436632 2024-11-22T23:17:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/a9c09dee496b456ca04081960a8712d9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9 2024-11-22T23:17:03,429 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9, entries=150, sequenceid=167, filesize=30.4 K 2024-11-22T23:17:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/cbb5aecdca104d48aa0b427c293c058a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a 2024-11-22T23:17:03,432 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T23:17:03,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/87c98133cad645109d040c38ff436632 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632 2024-11-22T23:17:03,436 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632, entries=150, sequenceid=167, filesize=11.9 K 2024-11-22T23:17:03,437 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 924a94030c700005d123b5bb895e1d9d in 957ms, sequenceid=167, compaction requested=false 2024-11-22T23:17:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:03,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-22T23:17:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-22T23:17:03,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-22T23:17:03,439 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6380 sec 2024-11-22T23:17:03,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.6420 sec 2024-11-22T23:17:03,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:03,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:03,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:03,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bacb2232444a4a469f472ae1ca39f605_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:03,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742389_1565 (size=14794) 2024-11-22T23:17:03,638 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:03,641 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122bacb2232444a4a469f472ae1ca39f605_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bacb2232444a4a469f472ae1ca39f605_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:03,643 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/881dc3eea0204a7caca1619784c80bdb, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:03,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/881dc3eea0204a7caca1619784c80bdb is 175, key is test_row_0/A:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742390_1566 (size=39749) 2024-11-22T23:17:03,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/881dc3eea0204a7caca1619784c80bdb 2024-11-22T23:17:03,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 is 50, key is test_row_0/B:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:03,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317483654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317483658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317483660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742391_1567 (size=12151) 2024-11-22T23:17:03,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317483767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317483768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317483777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317483977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317483977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:03,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:03,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317483988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:03,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 2024-11-22T23:17:04,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6232865d50f341a9a72d87b731a893c7 is 50, key is test_row_0/C:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742392_1568 (size=12151) 2024-11-22T23:17:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317484287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317484287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317484297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6232865d50f341a9a72d87b731a893c7 2024-11-22T23:17:04,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/881dc3eea0204a7caca1619784c80bdb as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb 2024-11-22T23:17:04,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb, entries=200, sequenceid=183, filesize=38.8 K 2024-11-22T23:17:04,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 2024-11-22T23:17:04,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4, entries=150, sequenceid=183, filesize=11.9 K 2024-11-22T23:17:04,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6232865d50f341a9a72d87b731a893c7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7 2024-11-22T23:17:04,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7, entries=150, sequenceid=183, filesize=11.9 K 2024-11-22T23:17:04,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 924a94030c700005d123b5bb895e1d9d in 897ms, sequenceid=183, compaction requested=true 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:04,517 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:17:04,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-22T23:17:04,517 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:04,518 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:04,518 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:17:04,518 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:04,518 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=99.9 K 2024-11-22T23:17:04,519 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:04,519 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb] 2024-11-22T23:17:04,521 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:04,521 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:17:04,521 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:04,521 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/071008cb46b443f1b6a3e9718b116cdb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.9 K 2024-11-22T23:17:04,521 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 25853199ebbc405eb5ff99b11ca2294e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:04,522 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071008cb46b443f1b6a3e9718b116cdb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:04,522 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c09dee496b456ca04081960a8712d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732317420806 2024-11-22T23:17:04,524 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbb5aecdca104d48aa0b427c293c058a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732317420806 2024-11-22T23:17:04,525 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 881dc3eea0204a7caca1619784c80bdb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:04,525 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe72b92ecca4da4bc4eafbe4d7d9ff4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:04,547 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#485 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:04,552 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/f69e78805b534d628ba3bca560e37d24 is 50, key is test_row_0/B:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:04,553 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:04,573 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411225fe5f1b3f2ad4611ade24d81cef4a711_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:04,575 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411225fe5f1b3f2ad4611ade24d81cef4a711_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:04,575 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411225fe5f1b3f2ad4611ade24d81cef4a711_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:04,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742393_1569 (size=12561) 2024-11-22T23:17:04,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742394_1570 (size=4469) 2024-11-22T23:17:04,597 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#486 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:04,597 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/96eaccf69db34c6e85d89652236222a8 is 175, key is test_row_0/A:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:04,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742395_1571 (size=31515) 2024-11-22T23:17:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:04,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:04,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112253e1b3cd1114401689016bcd5359cbac_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317424799/Put/seqid=0 2024-11-22T23:17:04,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742396_1572 (size=17284) 2024-11-22T23:17:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317484815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317484815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317484817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317484927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317484927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317484927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:04,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-22T23:17:04,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:04,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:17:04,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-22T23:17:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:04,982 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:17:04,982 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/f69e78805b534d628ba3bca560e37d24 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/f69e78805b534d628ba3bca560e37d24 2024-11-22T23:17:04,983 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:17:04,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:17:04,987 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into f69e78805b534d628ba3bca560e37d24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:04,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:04,987 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=13, startTime=1732317424517; duration=0sec 2024-11-22T23:17:04,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:04,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:17:04,987 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:04,993 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:04,993 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:17:04,993 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:04,993 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/d747995948f74275ab4fa608390268a6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=35.9 K 2024-11-22T23:17:04,993 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting d747995948f74275ab4fa608390268a6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732317420147 2024-11-22T23:17:04,994 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c98133cad645109d040c38ff436632, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732317420806 2024-11-22T23:17:04,994 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6232865d50f341a9a72d87b731a893c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:05,000 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#488 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:05,001 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b96d673cf5e45eda78ff7a84f8792c2 is 50, key is test_row_0/C:col10/1732317423618/Put/seqid=0 2024-11-22T23:17:05,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742397_1573 (size=12561) 2024-11-22T23:17:05,014 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/96eaccf69db34c6e85d89652236222a8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8 2024-11-22T23:17:05,018 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 96eaccf69db34c6e85d89652236222a8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:05,018 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:05,018 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=13, startTime=1732317424517; duration=0sec 2024-11-22T23:17:05,018 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:05,018 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:17:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:05,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317485137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317485138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317485138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,219 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:05,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112253e1b3cd1114401689016bcd5359cbac_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112253e1b3cd1114401689016bcd5359cbac_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:05,222 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2583c18b43d24968a6919943739aaf47, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:05,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2583c18b43d24968a6919943739aaf47 is 175, key is test_row_0/A:col10/1732317424799/Put/seqid=0 2024-11-22T23:17:05,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742398_1574 (size=48389) 2024-11-22T23:17:05,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:05,407 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/3b96d673cf5e45eda78ff7a84f8792c2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b96d673cf5e45eda78ff7a84f8792c2 2024-11-22T23:17:05,413 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into 3b96d673cf5e45eda78ff7a84f8792c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:05,413 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:05,413 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=13, startTime=1732317424517; duration=0sec 2024-11-22T23:17:05,413 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:05,413 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:17:05,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317485448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317485448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317485448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:05,626 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2583c18b43d24968a6919943739aaf47 2024-11-22T23:17:05,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/13ece0b498324868b20da31671746a1a is 50, key is test_row_0/B:col10/1732317424799/Put/seqid=0 2024-11-22T23:17:05,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742399_1575 (size=12151) 2024-11-22T23:17:05,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/13ece0b498324868b20da31671746a1a 2024-11-22T23:17:05,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/ce6308b35a244dcbbb6e78ccc4262c83 is 50, key is test_row_0/C:col10/1732317424799/Put/seqid=0 2024-11-22T23:17:05,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742400_1576 (size=12151) 2024-11-22T23:17:05,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:05,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317485958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317485958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317485958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:05,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:06,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:06,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/ce6308b35a244dcbbb6e78ccc4262c83 2024-11-22T23:17:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/2583c18b43d24968a6919943739aaf47 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47 2024-11-22T23:17:06,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47, entries=250, sequenceid=204, filesize=47.3 K 2024-11-22T23:17:06,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/13ece0b498324868b20da31671746a1a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a 2024-11-22T23:17:06,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T23:17:06,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/ce6308b35a244dcbbb6e78ccc4262c83 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83 2024-11-22T23:17:06,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83, entries=150, sequenceid=204, filesize=11.9 K 2024-11-22T23:17:06,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 924a94030c700005d123b5bb895e1d9d in 1305ms, sequenceid=204, compaction requested=false 2024-11-22T23:17:06,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:06,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:06,204 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227d71770627f04be59cfeae9de82ff77a_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317424816/Put/seqid=0 2024-11-22T23:17:06,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742401_1577 (size=12304) 2024-11-22T23:17:06,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:06,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317486281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317426270, max timestamp=1732317426283Num uniques 859; 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317426270, max timestamp=1732317426283Num uniques 859; 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732317426270, max timestamp=1732317426283Num uniques 859; 2024-11-22T23:17:06,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-22T23:17:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317486366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317486397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317486478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317486607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:06,630 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411227d71770627f04be59cfeae9de82ff77a_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d71770627f04be59cfeae9de82ff77a_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/bb5921bb7b4b4d7496850e0c91a3e1dd, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/bb5921bb7b4b4d7496850e0c91a3e1dd is 175, key is test_row_0/A:col10/1732317424816/Put/seqid=0 2024-11-22T23:17:06,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742402_1578 (size=31105) 2024-11-22T23:17:06,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317486687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317486917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317486967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317486967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,971 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:06,971 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:06,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317486977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:06,981 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:06,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317487000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:07,041 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/bb5921bb7b4b4d7496850e0c91a3e1dd 2024-11-22T23:17:07,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/8a0f92e24b464a289e98e0fc46fda809 is 50, key is test_row_0/B:col10/1732317424816/Put/seqid=0 2024-11-22T23:17:07,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742403_1579 (size=12151) 2024-11-22T23:17:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:07,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317487427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:07,449 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/8a0f92e24b464a289e98e0fc46fda809 2024-11-22T23:17:07,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/bb91626cf95440e1859f5d5304f0fea5 is 50, key is test_row_0/C:col10/1732317424816/Put/seqid=0 2024-11-22T23:17:07,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742404_1580 (size=12151) 2024-11-22T23:17:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317487508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:07,866 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/bb91626cf95440e1859f5d5304f0fea5 2024-11-22T23:17:07,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/bb5921bb7b4b4d7496850e0c91a3e1dd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd 2024-11-22T23:17:07,871 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd, entries=150, sequenceid=218, filesize=30.4 K 2024-11-22T23:17:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/8a0f92e24b464a289e98e0fc46fda809 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809 2024-11-22T23:17:07,874 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809, entries=150, sequenceid=218, filesize=11.9 K 2024-11-22T23:17:07,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/bb91626cf95440e1859f5d5304f0fea5 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5 2024-11-22T23:17:07,877 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5, entries=150, sequenceid=218, filesize=11.9 K 2024-11-22T23:17:07,878 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 924a94030c700005d123b5bb895e1d9d in 1674ms, sequenceid=218, compaction requested=true 2024-11-22T23:17:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:07,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-22T23:17:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-22T23:17:07,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-22T23:17:07,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8970 sec 2024-11-22T23:17:07,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.9030 sec 2024-11-22T23:17:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:08,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:08,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-22T23:17:08,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112280c84267222f4478b42dda5d3c4970d4_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:08,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742405_1581 (size=14794) 2024-11-22T23:17:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317488490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317488517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,521 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317488598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317488817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,862 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112280c84267222f4478b42dda5d3c4970d4_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280c84267222f4478b42dda5d3c4970d4_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:08,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4199c4e20ee644e38e1da86af65e629d, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:08,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4199c4e20ee644e38e1da86af65e629d is 175, key is test_row_0/A:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:08,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742406_1582 (size=39749) 2024-11-22T23:17:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317488988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317488988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,991 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:08,991 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:08,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317488997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:08,999 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:09,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:09,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317489127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-22T23:17:09,148 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:09,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-22T23:17:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-22T23:17:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:09,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-22T23:17:09,150 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-22T23:17:09,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-22T23:17:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:09,270 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4199c4e20ee644e38e1da86af65e629d 2024-11-22T23:17:09,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/888d927a56904d79ab37411f1cd63611 is 50, key is test_row_0/B:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:09,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742407_1583 (size=12151) 2024-11-22T23:17:09,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T23:17:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T23:17:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:09,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T23:17:09,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:09,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:17:09,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317489637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:09,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/888d927a56904d79ab37411f1cd63611 2024-11-22T23:17:09,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6f110fc122b34f42b21dbcddf833af8c is 50, key is test_row_0/C:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:09,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742408_1584 (size=12151) 2024-11-22T23:17:09,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6f110fc122b34f42b21dbcddf833af8c 2024-11-22T23:17:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4199c4e20ee644e38e1da86af65e629d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d 2024-11-22T23:17:09,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d, entries=200, sequenceid=254, filesize=38.8 K 2024-11-22T23:17:09,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/888d927a56904d79ab37411f1cd63611 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611 2024-11-22T23:17:09,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611, entries=150, sequenceid=254, filesize=11.9 K 2024-11-22T23:17:09,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/6f110fc122b34f42b21dbcddf833af8c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c 2024-11-22T23:17:09,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c, entries=150, sequenceid=254, filesize=11.9 K 2024-11-22T23:17:09,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 924a94030c700005d123b5bb895e1d9d in 1251ms, sequenceid=254, compaction requested=true 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:09,702 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:17:09,702 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:17:09,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:09,703 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:17:09,703 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:17:09,703 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,703 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=147.2 K 2024-11-22T23:17:09,704 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d] 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:17:09,704 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96eaccf69db34c6e85d89652236222a8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:09,704 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/f69e78805b534d628ba3bca560e37d24, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=47.9 K 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2583c18b43d24968a6919943739aaf47, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732317423632 2024-11-22T23:17:09,704 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting f69e78805b534d628ba3bca560e37d24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:09,705 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5921bb7b4b4d7496850e0c91a3e1dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732317424813 2024-11-22T23:17:09,705 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ece0b498324868b20da31671746a1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732317423657 2024-11-22T23:17:09,705 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4199c4e20ee644e38e1da86af65e629d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:09,705 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0f92e24b464a289e98e0fc46fda809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732317424813 2024-11-22T23:17:09,705 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 888d927a56904d79ab37411f1cd63611, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:09,710 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:09,712 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241122c7251ff58aae42fd959d62bf39a363ac_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:09,713 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:09,713 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/fee2bfc01fb24d09a0e467725cb4fbbd is 50, key is test_row_0/B:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:09,715 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241122c7251ff58aae42fd959d62bf39a363ac_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:09,715 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122c7251ff58aae42fd959d62bf39a363ac_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:09,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742409_1585 (size=12697) 2024-11-22T23:17:09,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742410_1586 (size=4469) 2024-11-22T23:17:09,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46715 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:09,760 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:09,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:09,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d957eb5a78149429c3ba313137dbce2_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317428488/Put/seqid=0 2024-11-22T23:17:09,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742411_1587 (size=12454) 2024-11-22T23:17:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:09,808 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T23:17:10,137 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/fee2bfc01fb24d09a0e467725cb4fbbd as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/fee2bfc01fb24d09a0e467725cb4fbbd 2024-11-22T23:17:10,141 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into fee2bfc01fb24d09a0e467725cb4fbbd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:10,141 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:10,141 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=12, startTime=1732317429702; duration=0sec 2024-11-22T23:17:10,141 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:10,141 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:17:10,141 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-22T23:17:10,143 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#497 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:10,143 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-22T23:17:10,143 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:17:10,143 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:10,143 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b96d673cf5e45eda78ff7a84f8792c2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=47.9 K 2024-11-22T23:17:10,143 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4f4802ce0c3c4e7f9e66beb8e2134517 is 175, key is test_row_0/A:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:10,144 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b96d673cf5e45eda78ff7a84f8792c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732317422981 2024-11-22T23:17:10,144 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting ce6308b35a244dcbbb6e78ccc4262c83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732317423657 2024-11-22T23:17:10,144 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting bb91626cf95440e1859f5d5304f0fea5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732317424813 2024-11-22T23:17:10,144 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f110fc122b34f42b21dbcddf833af8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742412_1588 (size=31651) 2024-11-22T23:17:10,152 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:10,152 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4f4802ce0c3c4e7f9e66beb8e2134517 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517 2024-11-22T23:17:10,152 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/7d80f76db7e840ae935d7c87c6db005e is 50, key is test_row_0/C:col10/1732317426312/Put/seqid=0 2024-11-22T23:17:10,156 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 4f4802ce0c3c4e7f9e66beb8e2134517(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:10,156 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:10,156 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=12, startTime=1732317429702; duration=0sec 2024-11-22T23:17:10,156 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:10,156 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:17:10,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742413_1589 (size=12697) 2024-11-22T23:17:10,161 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/7d80f76db7e840ae935d7c87c6db005e as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/7d80f76db7e840ae935d7c87c6db005e 2024-11-22T23:17:10,164 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into 7d80f76db7e840ae935d7c87c6db005e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:10,164 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:10,164 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=12, startTime=1732317429702; duration=0sec 2024-11-22T23:17:10,164 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:10,164 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:17:10,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:10,179 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411222d957eb5a78149429c3ba313137dbce2_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d957eb5a78149429c3ba313137dbce2_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:10,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/17378ca4cd964053900fe65e8f3bc15a, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:10,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/17378ca4cd964053900fe65e8f3bc15a is 175, key is test_row_0/A:col10/1732317428488/Put/seqid=0 2024-11-22T23:17:10,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742414_1590 (size=31255) 2024-11-22T23:17:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:10,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. as already flushing 2024-11-22T23:17:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:10,583 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=264, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/17378ca4cd964053900fe65e8f3bc15a 2024-11-22T23:17:10,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/95c77778464d465f84078083eeeb5403 is 50, key is test_row_0/B:col10/1732317428488/Put/seqid=0 2024-11-22T23:17:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742415_1591 (size=12301) 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317430603, max timestamp=1732317430620Num uniques 859; 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317430603, max timestamp=1732317430620Num uniques 859; 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732317430603, max timestamp=1732317430620Num uniques 859; 2024-11-22T23:17:10,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/a3d87a7361f3:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-22T23:17:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317490676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:10,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317490682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:10,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317490787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317490798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:10,991 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/95c77778464d465f84078083eeeb5403 2024-11-22T23:17:10,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/abcd574c151b444c86e9e1f335df70e3 is 50, key is test_row_0/C:col10/1732317428488/Put/seqid=0 2024-11-22T23:17:10,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742416_1592 (size=12301) 2024-11-22T23:17:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317490997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317491008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:11,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317491307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317491317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,399 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/abcd574c151b444c86e9e1f335df70e3 2024-11-22T23:17:11,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/17378ca4cd964053900fe65e8f3bc15a as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a 2024-11-22T23:17:11,410 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a, entries=150, sequenceid=264, filesize=30.5 K 2024-11-22T23:17:11,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/95c77778464d465f84078083eeeb5403 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403 2024-11-22T23:17:11,415 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403, entries=150, sequenceid=264, filesize=12.0 K 2024-11-22T23:17:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/abcd574c151b444c86e9e1f335df70e3 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3 2024-11-22T23:17:11,420 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3, entries=150, sequenceid=264, filesize=12.0 K 2024-11-22T23:17:11,421 INFO [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 924a94030c700005d123b5bb895e1d9d in 1660ms, sequenceid=264, compaction requested=false 2024-11-22T23:17:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:11,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-22T23:17:11,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-22T23:17:11,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-22T23:17:11,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2730 sec 2024-11-22T23:17:11,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 2.2770 sec 2024-11-22T23:17:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:11,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:11,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-22T23:17:11,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223c632a21e668481db231b3035238437f_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:11,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742417_1593 (size=14994) 2024-11-22T23:17:11,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317491839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317491844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317491958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:11,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317491958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:11,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317492168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317492168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,230 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223c632a21e668481db231b3035238437f_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c632a21e668481db231b3035238437f_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:12,234 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/9f8cd691a82f4d6cad5ba404d85aa2c2, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:12,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/9f8cd691a82f4d6cad5ba404d85aa2c2 is 175, key is test_row_0/A:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:12,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742418_1594 (size=39949) 2024-11-22T23:17:12,316 INFO [Thread-2349 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2349. 2024-11-22T23:17:12,316 DEBUG [Thread-2349 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:12,316 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,316 DEBUG [Thread-2349 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:12,318 INFO [Thread-2351 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2351. 2024-11-22T23:17:12,318 DEBUG [Thread-2351 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:12,318 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,318 DEBUG [Thread-2351 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:12,319 INFO [Thread-2343 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2343. 2024-11-22T23:17:12,319 DEBUG [Thread-2343 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:12,319 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,319 DEBUG [Thread-2343 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:12,320 INFO [Thread-2345 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2345. 2024-11-22T23:17:12,320 DEBUG [Thread-2345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:12,320 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,320 DEBUG [Thread-2345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:12,322 INFO [Thread-2347 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2347. 2024-11-22T23:17:12,322 DEBUG [Thread-2347 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:12,322 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,322 DEBUG [Thread-2347 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:12,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:12,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317492478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317492478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,638 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/9f8cd691a82f4d6cad5ba404d85aa2c2 2024-11-22T23:17:12,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/dbc8213334c94f4dbbd9837b935bd8a7 is 50, key is test_row_0/B:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:12,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742419_1595 (size=12301) 2024-11-22T23:17:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49378 deadline: 1732317492987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1732317492987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:13,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:13,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49458 deadline: 1732317493028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49402 deadline: 1732317493028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:13,028 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:13,028 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_1' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:13,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T23:17:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49432 deadline: 1732317493037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:13,038 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to a3d87a7361f3,46715,1732317251834 for 'test_row_0' in 924a94030c700005d123b5bb895e1d9d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] ... 32 more 2024-11-22T23:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 , the old value is region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-22T23:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., hostname=a3d87a7361f3,46715,1732317251834, seqNum=5 because the exception is null or not the one we care about 2024-11-22T23:17:13,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/dbc8213334c94f4dbbd9837b935bd8a7 2024-11-22T23:17:13,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ee57214d8f6461e9669d9b802c5552d is 50, key is test_row_0/C:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:13,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742420_1596 (size=12301) 2024-11-22T23:17:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-22T23:17:13,318 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:13,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ee57214d8f6461e9669d9b802c5552d 2024-11-22T23:17:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/9f8cd691a82f4d6cad5ba404d85aa2c2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2 2024-11-22T23:17:13,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2, entries=200, sequenceid=305, filesize=39.0 K 2024-11-22T23:17:13,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/dbc8213334c94f4dbbd9837b935bd8a7 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7 2024-11-22T23:17:13,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7, entries=150, sequenceid=305, filesize=12.0 K 2024-11-22T23:17:13,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/0ee57214d8f6461e9669d9b802c5552d as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d 2024-11-22T23:17:13,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d, entries=150, sequenceid=305, filesize=12.0 K 2024-11-22T23:17:13,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 924a94030c700005d123b5bb895e1d9d in 1656ms, sequenceid=305, compaction requested=true 2024-11-22T23:17:13,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:A, priority=-2147483648, current under compaction store size is 1 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:B, priority=-2147483648, current under compaction store size is 2 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 924a94030c700005d123b5bb895e1d9d:C, priority=-2147483648, current under compaction store size is 3 2024-11-22T23:17:13,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/A is initiating minor compaction (all files) 2024-11-22T23:17:13,477 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/B is initiating minor compaction (all files) 2024-11-22T23:17:13,477 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/A in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:13,477 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/B in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:13,477 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=100.4 K 2024-11-22T23:17:13,478 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/fee2bfc01fb24d09a0e467725cb4fbbd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=36.4 K 2024-11-22T23:17:13,478 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. files: [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2] 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting fee2bfc01fb24d09a0e467725cb4fbbd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f4802ce0c3c4e7f9e66beb8e2134517, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17378ca4cd964053900fe65e8f3bc15a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732317428465 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c77778464d465f84078083eeeb5403, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732317428465 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f8cd691a82f4d6cad5ba404d85aa2c2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317430668 2024-11-22T23:17:13,478 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting dbc8213334c94f4dbbd9837b935bd8a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317430674 2024-11-22T23:17:13,482 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:13,483 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:13,483 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411223cef4e03b0d34c2da0b964a80694ebec_924a94030c700005d123b5bb895e1d9d store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:13,484 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/da954b15e8924783b47d0686f921d760 is 50, key is test_row_0/B:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:13,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742421_1597 (size=12949) 2024-11-22T23:17:13,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411223cef4e03b0d34c2da0b964a80694ebec_924a94030c700005d123b5bb895e1d9d, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:13,491 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411223cef4e03b0d34c2da0b964a80694ebec_924a94030c700005d123b5bb895e1d9d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:13,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742422_1598 (size=4469) 2024-11-22T23:17:13,892 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/da954b15e8924783b47d0686f921d760 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/da954b15e8924783b47d0686f921d760 2024-11-22T23:17:13,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/B of 924a94030c700005d123b5bb895e1d9d into da954b15e8924783b47d0686f921d760(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:13,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:13,896 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/B, priority=13, startTime=1732317433477; duration=0sec 2024-11-22T23:17:13,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-22T23:17:13,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:B 2024-11-22T23:17:13,896 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-22T23:17:13,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-22T23:17:13,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1541): 924a94030c700005d123b5bb895e1d9d/C is initiating minor compaction (all files) 2024-11-22T23:17:13,897 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 924a94030c700005d123b5bb895e1d9d/C in TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:13,897 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/7d80f76db7e840ae935d7c87c6db005e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d] into tmpdir=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp, totalSize=36.4 K 2024-11-22T23:17:13,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d80f76db7e840ae935d7c87c6db005e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732317426312 2024-11-22T23:17:13,897 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting abcd574c151b444c86e9e1f335df70e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1732317428465 2024-11-22T23:17:13,898 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ee57214d8f6461e9669d9b802c5552d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732317430674 2024-11-22T23:17:13,900 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:13,900 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4656f94980c74481945b076eb51e9e8c is 175, key is test_row_0/A:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:13,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742423_1599 (size=31903) 2024-11-22T23:17:13,904 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 924a94030c700005d123b5bb895e1d9d#C#compaction#508 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-22T23:17:13,904 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/c315956e48e24943a2403f7a08711ab6 is 50, key is test_row_0/C:col10/1732317430674/Put/seqid=0 2024-11-22T23:17:13,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742424_1600 (size=12949) 2024-11-22T23:17:13,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46715 {}] regionserver.HRegion(8855): Flush requested on 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:13,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-22T23:17:13,999 INFO [Thread-2334 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2334. 2024-11-22T23:17:13,999 INFO [Thread-2332 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2332. 2024-11-22T23:17:13,999 DEBUG [Thread-2332 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:13,999 DEBUG [Thread-2334 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:13,999 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:13,999 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:13,999 DEBUG [Thread-2332 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:13,999 DEBUG [Thread-2334 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:13,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:13,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:13,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:14,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:14,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:14,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:14,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:14,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122980b7b7c519a465daa3e6060c02c1cd8_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_1/A:col10/1732317431837/Put/seqid=0 2024-11-22T23:17:14,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742425_1601 (size=9914) 2024-11-22T23:17:14,308 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/4656f94980c74481945b076eb51e9e8c as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4656f94980c74481945b076eb51e9e8c 2024-11-22T23:17:14,312 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/A of 924a94030c700005d123b5bb895e1d9d into 4656f94980c74481945b076eb51e9e8c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:14,312 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:14,312 INFO [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/A, priority=13, startTime=1732317433476; duration=0sec 2024-11-22T23:17:14,312 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:14,312 DEBUG [RS:0;a3d87a7361f3:46715-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:A 2024-11-22T23:17:14,312 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/c315956e48e24943a2403f7a08711ab6 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/c315956e48e24943a2403f7a08711ab6 2024-11-22T23:17:14,315 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 924a94030c700005d123b5bb895e1d9d/C of 924a94030c700005d123b5bb895e1d9d into c315956e48e24943a2403f7a08711ab6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-22T23:17:14,316 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:14,316 INFO [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., storeName=924a94030c700005d123b5bb895e1d9d/C, priority=13, startTime=1732317433477; duration=0sec 2024-11-22T23:17:14,316 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-22T23:17:14,316 DEBUG [RS:0;a3d87a7361f3:46715-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 924a94030c700005d123b5bb895e1d9d:C 2024-11-22T23:17:14,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:14,411 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122980b7b7c519a465daa3e6060c02c1cd8_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122980b7b7c519a465daa3e6060c02c1cd8_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:14,411 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/d3ef87b3a5864b18ab4bcd977d632671, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:14,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/d3ef87b3a5864b18ab4bcd977d632671 is 175, key is test_row_1/A:col10/1732317431837/Put/seqid=0 2024-11-22T23:17:14,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742426_1602 (size=22561) 2024-11-22T23:17:14,815 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/d3ef87b3a5864b18ab4bcd977d632671 2024-11-22T23:17:14,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1257a175b2a430a964895a9d90814ad is 50, key is test_row_1/B:col10/1732317431837/Put/seqid=0 2024-11-22T23:17:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742427_1603 (size=9857) 2024-11-22T23:17:15,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1257a175b2a430a964895a9d90814ad 2024-11-22T23:17:15,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/954cb0ff55ab432bbe61fa3d6839b4b9 is 50, key is test_row_1/C:col10/1732317431837/Put/seqid=0 2024-11-22T23:17:15,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742428_1604 (size=9857) 2024-11-22T23:17:15,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/954cb0ff55ab432bbe61fa3d6839b4b9 2024-11-22T23:17:15,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/d3ef87b3a5864b18ab4bcd977d632671 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/d3ef87b3a5864b18ab4bcd977d632671 2024-11-22T23:17:15,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/d3ef87b3a5864b18ab4bcd977d632671, entries=100, sequenceid=317, filesize=22.0 K 2024-11-22T23:17:15,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/b1257a175b2a430a964895a9d90814ad as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1257a175b2a430a964895a9d90814ad 2024-11-22T23:17:15,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1257a175b2a430a964895a9d90814ad, entries=100, sequenceid=317, filesize=9.6 K 2024-11-22T23:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/954cb0ff55ab432bbe61fa3d6839b4b9 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/954cb0ff55ab432bbe61fa3d6839b4b9 2024-11-22T23:17:15,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/954cb0ff55ab432bbe61fa3d6839b4b9, entries=100, sequenceid=317, filesize=9.6 K 2024-11-22T23:17:15,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 924a94030c700005d123b5bb895e1d9d in 1646ms, sequenceid=317, compaction requested=false 2024-11-22T23:17:15,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 924a94030c700005d123b5bb895e1d9d: 2024-11-22T23:17:23,049 INFO [Thread-2340 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2340. 2024-11-22T23:17:23,049 DEBUG [Thread-2340 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:23,049 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,049 DEBUG [Thread-2340 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:23,109 INFO [Thread-2338 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2338. 2024-11-22T23:17:23,109 INFO [Thread-2336 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2336. 2024-11-22T23:17:23,109 DEBUG [Thread-2338 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:23,109 DEBUG [Thread-2336 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-22T23:17:23,109 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,109 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,109 DEBUG [Thread-2338 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,109 DEBUG [Thread-2336 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:23,109 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-22T23:17:23,109 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2969 2024-11-22T23:17:23,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2978 2024-11-22T23:17:23,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2947 2024-11-22T23:17:23,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2971 2024-11-22T23:17:23,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2988 2024-11-22T23:17:23,110 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-22T23:17:23,110 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:17:23,110 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:17:23,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:23,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:23,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-22T23:17:23,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:23,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:23,113 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317443113"}]},"ts":"1732317443113"} 2024-11-22T23:17:23,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-22T23:17:23,114 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-22T23:17:23,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-22T23:17:23,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, UNASSIGN}] 2024-11-22T23:17:23,116 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, UNASSIGN 2024-11-22T23:17:23,117 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=CLOSING, regionLocation=a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:23,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, UNASSIGN because future has completed 2024-11-22T23:17:23,118 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-22T23:17:23,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834}] 2024-11-22T23:17:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:23,271 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 924a94030c700005d123b5bb895e1d9d, disabling compactions & flushes 2024-11-22T23:17:23,271 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. after waiting 0 ms 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:23,271 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 924a94030c700005d123b5bb895e1d9d 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=A 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=B 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 924a94030c700005d123b5bb895e1d9d, store=C 2024-11-22T23:17:23,271 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-22T23:17:23,276 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6e99a56d9a14f208a5cb23433def30c_924a94030c700005d123b5bb895e1d9d is 50, key is test_row_0/A:col10/1732317443108/Put/seqid=0 2024-11-22T23:17:23,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742429_1605 (size=9914) 2024-11-22T23:17:23,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:23,679 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:17:23,683 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241122e6e99a56d9a14f208a5cb23433def30c_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6e99a56d9a14f208a5cb23433def30c_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:23,684 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/5b591bd43cf041a190a32a1c0cd33827, store: [table=TestAcidGuarantees family=A region=924a94030c700005d123b5bb895e1d9d] 2024-11-22T23:17:23,684 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/5b591bd43cf041a190a32a1c0cd33827 is 175, key is test_row_0/A:col10/1732317443108/Put/seqid=0 2024-11-22T23:17:23,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742430_1606 (size=22561) 2024-11-22T23:17:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:24,087 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/5b591bd43cf041a190a32a1c0cd33827 2024-11-22T23:17:24,092 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/a520e743c9e74d61b6cfd520ca138c77 is 50, key is test_row_0/B:col10/1732317443108/Put/seqid=0 2024-11-22T23:17:24,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742431_1607 (size=9857) 2024-11-22T23:17:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:24,496 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/a520e743c9e74d61b6cfd520ca138c77 2024-11-22T23:17:24,501 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/9db738cbb505450c971ea8956e63e0f8 is 50, key is test_row_0/C:col10/1732317443108/Put/seqid=0 2024-11-22T23:17:24,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742432_1608 (size=9857) 2024-11-22T23:17:24,904 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/9db738cbb505450c971ea8956e63e0f8 2024-11-22T23:17:24,908 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/A/5b591bd43cf041a190a32a1c0cd33827 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/5b591bd43cf041a190a32a1c0cd33827 2024-11-22T23:17:24,910 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/5b591bd43cf041a190a32a1c0cd33827, entries=100, sequenceid=325, filesize=22.0 K 2024-11-22T23:17:24,911 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/B/a520e743c9e74d61b6cfd520ca138c77 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/a520e743c9e74d61b6cfd520ca138c77 2024-11-22T23:17:24,913 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/a520e743c9e74d61b6cfd520ca138c77, entries=100, sequenceid=325, filesize=9.6 K 2024-11-22T23:17:24,914 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/.tmp/C/9db738cbb505450c971ea8956e63e0f8 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/9db738cbb505450c971ea8956e63e0f8 2024-11-22T23:17:24,917 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/9db738cbb505450c971ea8956e63e0f8, entries=100, sequenceid=325, filesize=9.6 K 2024-11-22T23:17:24,917 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 924a94030c700005d123b5bb895e1d9d in 1646ms, sequenceid=325, compaction requested=true 2024-11-22T23:17:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2] to archive 2024-11-22T23:17:24,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:17:24,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a37ba687ea1646c3b5594aee9a641ee5 2024-11-22T23:17:24,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/ea51667369674e85b798411b09fc24f8 2024-11-22T23:17:24,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2012a0e9aa58400281b4589bafc3e6a0 2024-11-22T23:17:24,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/56fba3496ac44c9f88470bdf35ea07f1 2024-11-22T23:17:24,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/767b929463e84143b5f5fd151258eabe 2024-11-22T23:17:24,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/7c499021a8e6432191140741e98d981f 2024-11-22T23:17:24,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4c8db159a29b4f5da7686a7a734afaf3 2024-11-22T23:17:24,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/3635c8d3111a4a4790250f3207c50cb6 2024-11-22T23:17:24,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/f91806e8f6214ebabbc34b30ae372a72 2024-11-22T23:17:24,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/25853199ebbc405eb5ff99b11ca2294e 2024-11-22T23:17:24,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/a9c09dee496b456ca04081960a8712d9 2024-11-22T23:17:24,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/881dc3eea0204a7caca1619784c80bdb 2024-11-22T23:17:24,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/96eaccf69db34c6e85d89652236222a8 2024-11-22T23:17:24,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/2583c18b43d24968a6919943739aaf47 2024-11-22T23:17:24,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/bb5921bb7b4b4d7496850e0c91a3e1dd 2024-11-22T23:17:24,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4199c4e20ee644e38e1da86af65e629d 2024-11-22T23:17:24,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4f4802ce0c3c4e7f9e66beb8e2134517 2024-11-22T23:17:24,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/17378ca4cd964053900fe65e8f3bc15a 2024-11-22T23:17:24,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/9f8cd691a82f4d6cad5ba404d85aa2c2 2024-11-22T23:17:24,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d9eabc642e4a4d51b5e1c6221893d753, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/148f8172f32145a5838775d09eb31520, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/071008cb46b443f1b6a3e9718b116cdb, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/f69e78805b534d628ba3bca560e37d24, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/fee2bfc01fb24d09a0e467725cb4fbbd, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7] to archive 2024-11-22T23:17:24,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:17:24,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d4241e668f054b80a2d6e1219f78b568 2024-11-22T23:17:24,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1e3f8fe95ba4d09868183128b72ce54 2024-11-22T23:17:24,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d9eabc642e4a4d51b5e1c6221893d753 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d9eabc642e4a4d51b5e1c6221893d753 2024-11-22T23:17:24,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/85019661c49d40e1abc83f61f0255589 2024-11-22T23:17:24,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/7433e489876e4831a1aece7b9e151c62 2024-11-22T23:17:24,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/148f8172f32145a5838775d09eb31520 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/148f8172f32145a5838775d09eb31520 2024-11-22T23:17:24,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/d6d9d152878c46f799bd5c5215928c78 2024-11-22T23:17:24,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/06cade093d404e17892aa21b8a8bf25b 2024-11-22T23:17:24,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/071008cb46b443f1b6a3e9718b116cdb to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/071008cb46b443f1b6a3e9718b116cdb 2024-11-22T23:17:24,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/bbe1f78f2e404790ba37c987b0767ba4 2024-11-22T23:17:24,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/cbb5aecdca104d48aa0b427c293c058a 2024-11-22T23:17:24,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/f69e78805b534d628ba3bca560e37d24 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/f69e78805b534d628ba3bca560e37d24 2024-11-22T23:17:24,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/abe72b92ecca4da4bc4eafbe4d7d9ff4 2024-11-22T23:17:24,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/13ece0b498324868b20da31671746a1a 2024-11-22T23:17:24,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/8a0f92e24b464a289e98e0fc46fda809 2024-11-22T23:17:24,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/fee2bfc01fb24d09a0e467725cb4fbbd to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/fee2bfc01fb24d09a0e467725cb4fbbd 2024-11-22T23:17:24,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/888d927a56904d79ab37411f1cd63611 2024-11-22T23:17:24,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/95c77778464d465f84078083eeeb5403 2024-11-22T23:17:24,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/dbc8213334c94f4dbbd9837b935bd8a7 2024-11-22T23:17:24,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b345b7d73c14cfe9c36fe6af0ee3ef0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0465c7998ffe4007a57b65a85621cca0, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/d747995948f74275ab4fa608390268a6, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b96d673cf5e45eda78ff7a84f8792c2, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/7d80f76db7e840ae935d7c87c6db005e, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d] to archive 2024-11-22T23:17:24,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-22T23:17:24,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/8d3db44bb3d6469fbbb2626aaa3685b4 2024-11-22T23:17:24,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/19a1f1a7772847209d7b19ffeb029bcc 2024-11-22T23:17:24,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b345b7d73c14cfe9c36fe6af0ee3ef0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b345b7d73c14cfe9c36fe6af0ee3ef0 2024-11-22T23:17:24,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/a6a1b58d2ad14514991b11e483daa47a 2024-11-22T23:17:24,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/4f2e5795e0ae43b9bbee1081abfb8eac 2024-11-22T23:17:24,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0465c7998ffe4007a57b65a85621cca0 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0465c7998ffe4007a57b65a85621cca0 2024-11-22T23:17:24,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ef01c683540431088b8c54558ea2adf 2024-11-22T23:17:24,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b000df59351405f957ff10f1b9e376d 2024-11-22T23:17:24,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/d747995948f74275ab4fa608390268a6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/d747995948f74275ab4fa608390268a6 2024-11-22T23:17:24,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/52109c4339c94bec9a1a312ca39f38e2 2024-11-22T23:17:24,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/87c98133cad645109d040c38ff436632 2024-11-22T23:17:24,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b96d673cf5e45eda78ff7a84f8792c2 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/3b96d673cf5e45eda78ff7a84f8792c2 2024-11-22T23:17:24,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6232865d50f341a9a72d87b731a893c7 2024-11-22T23:17:24,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/ce6308b35a244dcbbb6e78ccc4262c83 2024-11-22T23:17:24,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/bb91626cf95440e1859f5d5304f0fea5 2024-11-22T23:17:24,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/7d80f76db7e840ae935d7c87c6db005e to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/7d80f76db7e840ae935d7c87c6db005e 2024-11-22T23:17:24,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/6f110fc122b34f42b21dbcddf833af8c 2024-11-22T23:17:24,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/abcd574c151b444c86e9e1f335df70e3 2024-11-22T23:17:24,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/0ee57214d8f6461e9669d9b802c5552d 2024-11-22T23:17:24,965 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits/328.seqid, newMaxSeqId=328, maxSeqId=4 2024-11-22T23:17:24,966 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. 2024-11-22T23:17:24,966 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 924a94030c700005d123b5bb895e1d9d: Waiting for close lock at 1732317443271Running coprocessor pre-close hooks at 1732317443271Disabling compacts and flushes for region at 1732317443271Disabling writes for close at 1732317443271Obtaining lock to block concurrent updates at 1732317443271Preparing flush snapshotting stores in 924a94030c700005d123b5bb895e1d9d at 1732317443271Finished memstore snapshotting TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732317443271Flushing stores of TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d. at 1732317443272 (+1 ms)Flushing 924a94030c700005d123b5bb895e1d9d/A: creating writer at 1732317443272Flushing mob file 924a94030c700005d123b5bb895e1d9d/A: appending metadata at 1732317443275 (+3 ms)Flushing mob file 924a94030c700005d123b5bb895e1d9d/A: closing flushed file at 1732317443275Flushing 924a94030c700005d123b5bb895e1d9d/A: appending metadata at 1732317443684 (+409 ms)Flushing 924a94030c700005d123b5bb895e1d9d/A: closing flushed file at 1732317443684Flushing 924a94030c700005d123b5bb895e1d9d/B: creating writer at 1732317444090 (+406 ms)Flushing 924a94030c700005d123b5bb895e1d9d/B: appending metadata at 1732317444092 (+2 ms)Flushing 924a94030c700005d123b5bb895e1d9d/B: closing flushed file at 1732317444092Flushing 924a94030c700005d123b5bb895e1d9d/C: creating writer at 1732317444498 (+406 ms)Flushing 924a94030c700005d123b5bb895e1d9d/C: appending metadata at 1732317444501 (+3 ms)Flushing 924a94030c700005d123b5bb895e1d9d/C: closing flushed file at 1732317444501Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@524e215d: reopening flushed file at 1732317444907 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53193e29: reopening flushed file at 1732317444910 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8802baa: reopening flushed file at 1732317444913 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 924a94030c700005d123b5bb895e1d9d in 1646ms, sequenceid=325, compaction requested=true at 1732317444917 (+4 ms)Writing region close event to WAL at 1732317444962 (+45 ms)Running coprocessor post-close hooks at 1732317444966 (+4 ms)Closed at 1732317444966 2024-11-22T23:17:24,968 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:24,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=924a94030c700005d123b5bb895e1d9d, regionState=CLOSED 2024-11-22T23:17:24,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 because future has completed 2024-11-22T23:17:24,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-22T23:17:24,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 924a94030c700005d123b5bb895e1d9d, server=a3d87a7361f3,46715,1732317251834 in 1.8520 sec 2024-11-22T23:17:24,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-22T23:17:24,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=924a94030c700005d123b5bb895e1d9d, UNASSIGN in 1.8550 sec 2024-11-22T23:17:24,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-22T23:17:24,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8580 sec 2024-11-22T23:17:24,975 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317444975"}]},"ts":"1732317444975"} 2024-11-22T23:17:24,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-22T23:17:24,976 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-22T23:17:24,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8660 sec 2024-11-22T23:17:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-22T23:17:25,257 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:25,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-22T23:17:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,259 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T23:17:25,260 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,262 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C, FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits] 2024-11-22T23:17:25,265 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4656f94980c74481945b076eb51e9e8c to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/4656f94980c74481945b076eb51e9e8c 2024-11-22T23:17:25,266 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/5b591bd43cf041a190a32a1c0cd33827 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/5b591bd43cf041a190a32a1c0cd33827 2024-11-22T23:17:25,267 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/d3ef87b3a5864b18ab4bcd977d632671 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/A/d3ef87b3a5864b18ab4bcd977d632671 2024-11-22T23:17:25,268 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/a520e743c9e74d61b6cfd520ca138c77 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/a520e743c9e74d61b6cfd520ca138c77 2024-11-22T23:17:25,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1257a175b2a430a964895a9d90814ad to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/b1257a175b2a430a964895a9d90814ad 2024-11-22T23:17:25,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/da954b15e8924783b47d0686f921d760 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/B/da954b15e8924783b47d0686f921d760 2024-11-22T23:17:25,270 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/954cb0ff55ab432bbe61fa3d6839b4b9 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/954cb0ff55ab432bbe61fa3d6839b4b9 2024-11-22T23:17:25,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/9db738cbb505450c971ea8956e63e0f8 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/9db738cbb505450c971ea8956e63e0f8 2024-11-22T23:17:25,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/c315956e48e24943a2403f7a08711ab6 to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/C/c315956e48e24943a2403f7a08711ab6 2024-11-22T23:17:25,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits/328.seqid to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d/recovered.edits/328.seqid 2024-11-22T23:17:25,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/default/TestAcidGuarantees/924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,274 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-22T23:17:25,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:17:25,275 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-22T23:17:25,277 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221f000caa17eb40b2990240e4fe77030b_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411221f000caa17eb40b2990240e4fe77030b_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,278 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d957eb5a78149429c3ba313137dbce2_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411222d957eb5a78149429c3ba313137dbce2_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,279 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122348ca7fa9a4d4cf5b2d50ae2f6461d72_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122348ca7fa9a4d4cf5b2d50ae2f6461d72_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,280 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c632a21e668481db231b3035238437f_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411223c632a21e668481db231b3035238437f_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,280 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112251330bac21914dd8b22b6049bba120fd_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112251330bac21914dd8b22b6049bba120fd_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,281 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112253e1b3cd1114401689016bcd5359cbac_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112253e1b3cd1114401689016bcd5359cbac_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,282 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225befd91b95894750b65699d5cc279ff2_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225befd91b95894750b65699d5cc279ff2_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,283 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225cac39838c54436489d77c6e0b706976_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411225cac39838c54436489d77c6e0b706976_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,284 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264a1a9fa5601437a896c50dd31447957_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112264a1a9fa5601437a896c50dd31447957_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,284 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d71770627f04be59cfeae9de82ff77a_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411227d71770627f04be59cfeae9de82ff77a_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,285 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280c84267222f4478b42dda5d3c4970d4_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112280c84267222f4478b42dda5d3c4970d4_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,286 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122980b7b7c519a465daa3e6060c02c1cd8_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122980b7b7c519a465daa3e6060c02c1cd8_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,286 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bacb2232444a4a469f472ae1ca39f605_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122bacb2232444a4a469f472ae1ca39f605_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,287 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122cb95cd380a784ee4815fdeb98facfa86_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122cb95cd380a784ee4815fdeb98facfa86_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,287 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4a51b31f66e45fd88a7b40118344da1_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e4a51b31f66e45fd88a7b40118344da1_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,288 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6e99a56d9a14f208a5cb23433def30c_924a94030c700005d123b5bb895e1d9d to hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241122e6e99a56d9a14f208a5cb23433def30c_924a94030c700005d123b5bb895e1d9d 2024-11-22T23:17:25,288 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-22T23:17:25,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,292 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-22T23:17:25,294 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-22T23:17:25,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,295 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-22T23:17:25,295 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732317445295"}]},"ts":"9223372036854775807"} 2024-11-22T23:17:25,296 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-22T23:17:25,296 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 924a94030c700005d123b5bb895e1d9d, NAME => 'TestAcidGuarantees,,1732317409187.924a94030c700005d123b5bb895e1d9d.', STARTKEY => '', ENDKEY => ''}] 2024-11-22T23:17:25,296 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-22T23:17:25,296 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732317445296"}]},"ts":"9223372036854775807"} 2024-11-22T23:17:25,298 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-22T23:17:25,298 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-22T23:17:25,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-22T23:17:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34971 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-22T23:17:25,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-22T23:17:25,367 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-22T23:17:25,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 249), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=404 (was 384) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=855 (was 1615) 2024-11-22T23:17:25,374 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:17:25,374 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:17:25,375 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:17:25,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:25,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:25,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:17:25,375 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:17:25,375 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-22T23:17:25,375 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3d87a7361f3,34971,1732317250724 2024-11-22T23:17:25,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:17:25,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:17:25,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:17:25,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:17:25,465 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:17:25,465 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:17:25,465 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:17:25,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:25,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:17:25,466 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3d87a7361f3,46715,1732317251834' ***** 2024-11-22T23:17:25,466 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T23:17:25,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:17:25,466 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:17:25,466 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:17:25,466 INFO [RS:0;a3d87a7361f3:46715 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(959): stopping server a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3d87a7361f3:46715. 2024-11-22T23:17:25,467 DEBUG [RS:0;a3d87a7361f3:46715 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:17:25,467 DEBUG [RS:0;a3d87a7361f3:46715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:17:25,467 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:17:25,468 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T23:17:25,468 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:17:25,468 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:17:25,468 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:17:25,468 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:17:25,468 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:25,468 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:17:25,468 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:17:25,468 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-22T23:17:25,474 INFO [regionserver/a3d87a7361f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T23:17:25,474 INFO [regionserver/a3d87a7361f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T23:17:25,489 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/info/030fc529a7404a8a92b40c13aa2a9987 is 95, key is TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7./info:/1732317283239/DeleteFamily/seqid=0 2024-11-22T23:17:25,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742433_1609 (size=6514) 2024-11-22T23:17:25,558 INFO [regionserver/a3d87a7361f3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:17:25,668 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:25,868 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:25,893 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/info/030fc529a7404a8a92b40c13aa2a9987 2024-11-22T23:17:25,910 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/ns/cd4bbe2ff4054f70b5d2641a44843232 is 93, key is TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7./ns:/1732317283198/DeleteFamily/seqid=0 2024-11-22T23:17:25,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742434_1610 (size=5906) 2024-11-22T23:17:26,068 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:26,269 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:26,313 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/ns/cd4bbe2ff4054f70b5d2641a44843232 2024-11-22T23:17:26,331 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/rep_barrier/96e255fe049e44999eccffb6c24822ce is 102, key is TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7./rep_barrier:/1732317283198/DeleteFamily/seqid=0 2024-11-22T23:17:26,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742435_1611 (size=6025) 2024-11-22T23:17:26,469 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:26,669 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:26,735 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/rep_barrier/96e255fe049e44999eccffb6c24822ce 2024-11-22T23:17:26,751 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/table/57ecdf3e447d4db4bcff3531ac076634 is 96, key is TestAcidGuarantees,,1732317256168.4a71851c051c02efdfd8dcb5ec6b9ff7./table:/1732317283198/DeleteFamily/seqid=0 2024-11-22T23:17:26,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742436_1612 (size=5936) 2024-11-22T23:17:26,869 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:27,069 DEBUG [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-22T23:17:27,155 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/table/57ecdf3e447d4db4bcff3531ac076634 2024-11-22T23:17:27,158 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/info/030fc529a7404a8a92b40c13aa2a9987 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/info/030fc529a7404a8a92b40c13aa2a9987 2024-11-22T23:17:27,161 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/info/030fc529a7404a8a92b40c13aa2a9987, entries=12, sequenceid=90, filesize=6.4 K 2024-11-22T23:17:27,162 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/ns/cd4bbe2ff4054f70b5d2641a44843232 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/ns/cd4bbe2ff4054f70b5d2641a44843232 2024-11-22T23:17:27,164 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/ns/cd4bbe2ff4054f70b5d2641a44843232, entries=8, sequenceid=90, filesize=5.8 K 2024-11-22T23:17:27,165 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/rep_barrier/96e255fe049e44999eccffb6c24822ce as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/rep_barrier/96e255fe049e44999eccffb6c24822ce 2024-11-22T23:17:27,167 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/rep_barrier/96e255fe049e44999eccffb6c24822ce, entries=6, sequenceid=90, filesize=5.9 K 2024-11-22T23:17:27,168 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/.tmp/table/57ecdf3e447d4db4bcff3531ac076634 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/table/57ecdf3e447d4db4bcff3531ac076634 2024-11-22T23:17:27,170 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/table/57ecdf3e447d4db4bcff3531ac076634, entries=7, sequenceid=90, filesize=5.8 K 2024-11-22T23:17:27,171 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false 2024-11-22T23:17:27,174 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-22T23:17:27,174 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:17:27,175 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:17:27,175 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732317445468Running coprocessor pre-close hooks at 1732317445468Disabling compacts and flushes for region at 1732317445468Disabling writes for close at 1732317445468Obtaining lock to block concurrent updates at 1732317445468Preparing flush snapshotting stores in 1588230740 at 1732317445468Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732317445468Flushing stores of hbase:meta,,1.1588230740 at 1732317445469 (+1 ms)Flushing 1588230740/info: creating writer at 1732317445469Flushing 1588230740/info: appending metadata at 1732317445489 (+20 ms)Flushing 1588230740/info: closing flushed file at 1732317445489Flushing 1588230740/ns: creating writer at 1732317445897 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732317445909 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1732317445909Flushing 1588230740/rep_barrier: creating writer at 1732317446316 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732317446331 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732317446331Flushing 1588230740/table: creating writer at 1732317446738 (+407 ms)Flushing 1588230740/table: appending metadata at 1732317446751 (+13 ms)Flushing 1588230740/table: closing flushed file at 1732317446751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a7e2e61: reopening flushed file at 1732317447158 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ddee190: reopening flushed file at 1732317447161 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ba2c4: reopening flushed file at 1732317447164 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4304927a: reopening flushed file at 1732317447167 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1703ms, sequenceid=90, compaction requested=false at 1732317447171 (+4 ms)Writing region close event to WAL at 1732317447172 (+1 ms)Running coprocessor post-close hooks at 1732317447174 (+2 ms)Closed at 1732317447174 2024-11-22T23:17:27,175 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:17:27,270 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(976): stopping server a3d87a7361f3,46715,1732317251834; all regions closed. 2024-11-22T23:17:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741834_1010 (size=25344) 2024-11-22T23:17:27,275 DEBUG [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/oldWALs 2024-11-22T23:17:27,275 INFO [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3d87a7361f3%2C46715%2C1732317251834.meta:.meta(num 1732317255664) 2024-11-22T23:17:27,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741832_1008 (size=17144032) 2024-11-22T23:17:27,277 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/WALs/a3d87a7361f3,46715,1732317251834/a3d87a7361f3%2C46715%2C1732317251834.1732317254608 not finished, retry = 0 2024-11-22T23:17:27,380 DEBUG [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/oldWALs 2024-11-22T23:17:27,380 INFO [RS:0;a3d87a7361f3:46715 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3d87a7361f3%2C46715%2C1732317251834:(num 1732317254608) 2024-11-22T23:17:27,380 DEBUG [RS:0;a3d87a7361f3:46715 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:17:27,380 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:17:27,380 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:17:27,380 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.ChoreService(370): Chore service for: regionserver/a3d87a7361f3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T23:17:27,380 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:17:27,380 INFO [regionserver/a3d87a7361f3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:17:27,381 INFO [RS:0;a3d87a7361f3:46715 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46715 2024-11-22T23:17:27,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3d87a7361f3,46715,1732317251834 2024-11-22T23:17:27,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:17:27,444 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:17:27,454 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3d87a7361f3,46715,1732317251834] 2024-11-22T23:17:27,465 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3d87a7361f3,46715,1732317251834 already deleted, retry=false 2024-11-22T23:17:27,465 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3d87a7361f3,46715,1732317251834 expired; onlineServers=0 2024-11-22T23:17:27,465 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a3d87a7361f3,34971,1732317250724' ***** 2024-11-22T23:17:27,465 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T23:17:27,465 INFO [M:0;a3d87a7361f3:34971 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:17:27,465 INFO [M:0;a3d87a7361f3:34971 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:17:27,465 DEBUG [M:0;a3d87a7361f3:34971 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:17:27,466 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:17:27,466 DEBUG [M:0;a3d87a7361f3:34971 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:17:27,466 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317254305 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317254305,5,FailOnTimeoutGroup] 2024-11-22T23:17:27,466 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317254306 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317254306,5,FailOnTimeoutGroup] 2024-11-22T23:17:27,466 INFO [M:0;a3d87a7361f3:34971 {}] hbase.ChoreService(370): Chore service for: master/a3d87a7361f3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:17:27,466 INFO [M:0;a3d87a7361f3:34971 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:17:27,466 DEBUG [M:0;a3d87a7361f3:34971 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:17:27,466 INFO [M:0;a3d87a7361f3:34971 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:17:27,466 INFO [M:0;a3d87a7361f3:34971 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:17:27,466 ERROR [M:0;a3d87a7361f3:34971 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:36519 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36519,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-22T23:17:27,467 INFO [M:0;a3d87a7361f3:34971 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:17:27,467 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:17:27,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:17:27,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:17:27,476 DEBUG [M:0;a3d87a7361f3:34971 {}] zookeeper.ZKUtil(347): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:17:27,476 WARN [M:0;a3d87a7361f3:34971 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:17:27,476 INFO [M:0;a3d87a7361f3:34971 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/.lastflushedseqids 2024-11-22T23:17:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742437_1613 (size=72) 2024-11-22T23:17:27,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:17:27,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46715-0x10164d37d070001, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:17:27,555 INFO [RS:0;a3d87a7361f3:46715 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:17:27,555 INFO [RS:0;a3d87a7361f3:46715 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3d87a7361f3,46715,1732317251834; zookeeper connection closed. 2024-11-22T23:17:27,556 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12646c43 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12646c43 2024-11-22T23:17:27,556 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T23:17:27,886 INFO [M:0;a3d87a7361f3:34971 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:17:27,886 INFO [M:0;a3d87a7361f3:34971 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:17:27,886 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:17:27,887 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:17:27,887 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:17:27,887 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:17:27,887 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:17:27,887 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=793.84 KB heapSize=973.78 KB 2024-11-22T23:17:27,904 DEBUG [M:0;a3d87a7361f3:34971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/213dcf6ce7df45b8be1268e87f8bac86 is 82, key is hbase:meta,,1/info:regioninfo/1732317255746/Put/seqid=0 2024-11-22T23:17:27,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742438_1614 (size=5672) 2024-11-22T23:17:28,307 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2226 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/213dcf6ce7df45b8be1268e87f8bac86 2024-11-22T23:17:28,328 DEBUG [M:0;a3d87a7361f3:34971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41056b04730843a88197f8e5960bcd07 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1732317412285/Put/seqid=0 2024-11-22T23:17:28,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742439_1615 (size=46396) 2024-11-22T23:17:28,732 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=793.29 KB at sequenceid=2226 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41056b04730843a88197f8e5960bcd07 2024-11-22T23:17:28,735 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41056b04730843a88197f8e5960bcd07 2024-11-22T23:17:28,748 DEBUG [M:0;a3d87a7361f3:34971 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5cd41d10b03c44fd85b245018e53eba2 is 69, key is a3d87a7361f3,46715,1732317251834/rs:state/1732317254369/Put/seqid=0 2024-11-22T23:17:28,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073742440_1616 (size=5156) 2024-11-22T23:17:29,152 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2226 (bloomFilter=true), to=hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5cd41d10b03c44fd85b245018e53eba2 2024-11-22T23:17:29,156 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/213dcf6ce7df45b8be1268e87f8bac86 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/213dcf6ce7df45b8be1268e87f8bac86 2024-11-22T23:17:29,158 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/213dcf6ce7df45b8be1268e87f8bac86, entries=8, sequenceid=2226, filesize=5.5 K 2024-11-22T23:17:29,159 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/41056b04730843a88197f8e5960bcd07 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41056b04730843a88197f8e5960bcd07 2024-11-22T23:17:29,162 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 41056b04730843a88197f8e5960bcd07 2024-11-22T23:17:29,162 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/41056b04730843a88197f8e5960bcd07, entries=176, sequenceid=2226, filesize=45.3 K 2024-11-22T23:17:29,162 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5cd41d10b03c44fd85b245018e53eba2 as hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5cd41d10b03c44fd85b245018e53eba2 2024-11-22T23:17:29,165 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36519/user/jenkins/test-data/91a6b159-7780-91e3-0ed9-282f86fcc35c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5cd41d10b03c44fd85b245018e53eba2, entries=1, sequenceid=2226, filesize=5.0 K 2024-11-22T23:17:29,165 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(3140): Finished flush of dataSize ~793.84 KB/812895, heapSize ~973.48 KB/996848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2226, compaction requested=false 2024-11-22T23:17:29,166 INFO [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:17:29,167 DEBUG [M:0;a3d87a7361f3:34971 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732317447886Disabling compacts and flushes for region at 1732317447886Disabling writes for close at 1732317447887 (+1 ms)Obtaining lock to block concurrent updates at 1732317447887Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732317447887Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=812895, getHeapSize=997088, getOffHeapSize=0, getCellsCount=2427 at 1732317447887Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732317447888 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732317447888Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732317447903 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732317447903Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732317448311 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732317448327 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732317448327Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732317448735 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732317448748 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732317448748Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@483baa3b: reopening flushed file at 1732317449155 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59c173e2: reopening flushed file at 1732317449158 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3825693e: reopening flushed file at 1732317449162 (+4 ms)Finished flush of dataSize ~793.84 KB/812895, heapSize ~973.48 KB/996848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2226, compaction requested=false at 1732317449165 (+3 ms)Writing region close event to WAL at 1732317449166 (+1 ms)Closed at 1732317449166 2024-11-22T23:17:29,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41203 is added to blk_1073741830_1006 (size=958808) 2024-11-22T23:17:29,169 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:17:29,170 INFO [M:0;a3d87a7361f3:34971 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:17:29,170 INFO [M:0;a3d87a7361f3:34971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34971 2024-11-22T23:17:29,170 INFO [M:0;a3d87a7361f3:34971 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:17:29,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:17:29,285 INFO [M:0;a3d87a7361f3:34971 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:17:29,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34971-0x10164d37d070000, quorum=127.0.0.1:62676, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:17:29,289 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T23:17:29,291 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:17:29,291 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:17:29,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:17:29,292 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.log.dir/,STOPPED} 2024-11-22T23:17:29,294 WARN [BP-579577094-172.17.0.2-1732317246869 heartbeating to localhost/127.0.0.1:36519 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T23:17:29,294 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T23:17:29,295 WARN [BP-579577094-172.17.0.2-1732317246869 heartbeating to localhost/127.0.0.1:36519 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-579577094-172.17.0.2-1732317246869 (Datanode Uuid df18387f-e6c9-4fc0-94ee-d6ae1ff9699b) service to localhost/127.0.0.1:36519 2024-11-22T23:17:29,295 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T23:17:29,296 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/data/data1/current/BP-579577094-172.17.0.2-1732317246869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:17:29,296 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/cluster_05182775-0688-0aaa-4d29-a56c4ffb6a2d/data/data2/current/BP-579577094-172.17.0.2-1732317246869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T23:17:29,296 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T23:17:29,307 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T23:17:29,308 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T23:17:29,308 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T23:17:29,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T23:17:29,308 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/527526b8-0d95-6a54-9241-1da1c27c3de7/hadoop.log.dir/,STOPPED} 2024-11-22T23:17:29,327 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:17:29,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down